diff --git a/CHANGELOG.md b/CHANGELOG.md index 712693ac3b..46c35721c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ -# 1.7.0 -* Rule miner that proposes data quality rule configuration -* Notifications can be filtered by various parameters, allowing to send notifications to the correct teams -* Estimated word count checks added to support analyzing data for GenAI use cases -* Some pattern validation checks renamed to use a unified naming convention "invalid" in the check names +# 1.8.0 + +* Small layout fixes in the UI, mostly related to the colors of issue severity colors +* Documentation for the data quality process and rule mining +* Duplicate record detection check +* Some column-level checks fixed to skip NULL values, instead of treating them as invalid (not passing checks) +* Store data lineage definition (links to source tables) +* Default observability check patterns renamed to data quality policies +* Default quality policy named "default" deleted, and replaced with multiple smaller policies to allow more control diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/__init__.py b/distribution/python/dqops/client/api/column_quality_policies/__init__.py similarity index 100% rename from distribution/python/dqops/client/api/default_column_check_patterns/__init__.py rename to distribution/python/dqops/client/api/column_quality_policies/__init__.py diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/copy_from_default_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/copy_from_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/copy_from_default_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/copy_from_column_quality_policy.py index 1d6570fa9e..8acf6c2c97 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/copy_from_default_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/copy_from_column_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "post", - "url": "api/default/checks/column/{targetPatternName}/copyfrom/{sourcePatternName}".format( + "url": "api/policies/checks/column/{targetPatternName}/copyfrom/{sourcePatternName}".format( targetPatternName=target_pattern_name, sourcePatternName=source_pattern_name, ), @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """copyFromDefaultColumnChecksPattern + """copyFromColumnQualityPolicy Creates (adds) a copy of an existing default column-level checks pattern configuration, under a new name. @@ -87,7 +87,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """copyFromDefaultColumnChecksPattern + """copyFromColumnQualityPolicy Creates (adds) a copy of an existing default column-level checks pattern configuration, under a new name. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy.py similarity index 82% rename from distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy.py index 6fabd8e1f7..b52211c70f 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy.py @@ -5,16 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_column_checks_pattern_model import ( - DefaultColumnChecksPatternModel, -) +from ...models.column_quality_policy_model import ColumnQualityPolicyModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultColumnChecksPatternModel, + json_body: ColumnQualityPolicyModel, ) -> Dict[str, Any]: pass @@ -23,7 +21,7 @@ def _get_kwargs( return { "method": "post", - "url": "api/default/checks/column/{patternName}".format( + "url": "api/policies/checks/column/{patternName}".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,16 +54,17 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternModel, + json_body: ColumnQualityPolicyModel, ) -> Response[Any]: - """createDefaultColumnChecksPattern + """createColumnQualityPolicy Creates (adds) a new default column-level checks pattern configuration by saving a full specification object. Args: pattern_name (str): - json_body (DefaultColumnChecksPatternModel): Default column-level checks pattern model + json_body (ColumnQualityPolicyModel): Default column-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -91,16 +90,17 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternModel, + json_body: ColumnQualityPolicyModel, ) -> Response[Any]: - """createDefaultColumnChecksPattern + """createColumnQualityPolicy Creates (adds) a new default column-level checks pattern configuration by saving a full specification object. Args: pattern_name (str): - json_body (DefaultColumnChecksPatternModel): Default column-level checks pattern model + json_body (ColumnQualityPolicyModel): Default column-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern_target.py b/distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy_target.py similarity index 79% rename from distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern_target.py rename to distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy_target.py index 24ac45b7e1..5004989f66 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern_target.py +++ b/distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy_target.py @@ -5,16 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_column_checks_pattern_list_model import ( - DefaultColumnChecksPatternListModel, -) +from ...models.column_quality_policy_list_model import ColumnQualityPolicyListModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultColumnChecksPatternListModel, + json_body: ColumnQualityPolicyListModel, ) -> Dict[str, Any]: pass @@ -23,7 +21,7 @@ def _get_kwargs( return { "method": "post", - "url": "api/default/checks/column/{patternName}/target".format( + "url": "api/policies/checks/column/{patternName}/target".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,16 +54,16 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternListModel, + json_body: ColumnQualityPolicyListModel, ) -> Response[Any]: - """createDefaultColumnChecksPatternTarget + """createColumnQualityPolicyTarget Creates (adds) a new default column-level checks pattern configuration. Args: pattern_name (str): - json_body (DefaultColumnChecksPatternListModel): Default column-level checks pattern list - model + json_body (ColumnQualityPolicyListModel): Default column-level checks pattern (data + quality policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -91,16 +89,16 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternListModel, + json_body: ColumnQualityPolicyListModel, ) -> Response[Any]: - """createDefaultColumnChecksPatternTarget + """createColumnQualityPolicyTarget Creates (adds) a new default column-level checks pattern configuration. Args: pattern_name (str): - json_body (DefaultColumnChecksPatternListModel): Default column-level checks pattern list - model + json_body (ColumnQualityPolicyListModel): Default column-level checks pattern (data + quality policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/delete_default_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/delete_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/delete_default_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/delete_column_quality_policy.py index c7e79e135e..694450fa78 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/delete_default_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/delete_column_quality_policy.py @@ -16,7 +16,7 @@ def _get_kwargs( return { "method": "delete", - "url": "api/default/checks/column/{patternName}".format( + "url": "api/policies/checks/column/{patternName}".format( patternName=pattern_name, ), } @@ -49,7 +49,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """deleteDefaultColumnChecksPattern + """deleteColumnQualityPolicy Deletes a default column-level checks pattern @@ -80,7 +80,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """deleteDefaultColumnChecksPattern + """deleteColumnQualityPolicy Deletes a default column-level checks pattern diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_all_default_column_checks_patterns.py b/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policies.py similarity index 77% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_all_default_column_checks_patterns.py rename to distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policies.py index eb1f8b1ecf..d544d95532 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_all_default_column_checks_patterns.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policies.py @@ -5,9 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_column_checks_pattern_list_model import ( - DefaultColumnChecksPatternListModel, -) +from ...models.column_quality_policy_list_model import ColumnQualityPolicyListModel from ...types import Response @@ -17,18 +15,18 @@ def _get_kwargs() -> Dict[str, Any]: return { "method": "get", - "url": "api/default/checks/column", + "url": "api/policies/checks/column", } def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[List["DefaultColumnChecksPatternListModel"]]: +) -> Optional[List["ColumnQualityPolicyListModel"]]: if response.status_code == HTTPStatus.OK: response_200 = [] _response_200 = response.json() for response_200_item_data in _response_200: - response_200_item = DefaultColumnChecksPatternListModel.from_dict( + response_200_item = ColumnQualityPolicyListModel.from_dict( response_200_item_data ) @@ -43,7 +41,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[List["DefaultColumnChecksPatternListModel"]]: +) -> Response[List["ColumnQualityPolicyListModel"]]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -55,8 +53,8 @@ def _build_response( def sync_detailed( *, client: AuthenticatedClient, -) -> Response[List["DefaultColumnChecksPatternListModel"]]: - """getAllDefaultColumnChecksPatterns +) -> Response[List["ColumnQualityPolicyListModel"]]: + """getColumnQualityPolicies Returns a flat list of all column-level default check patterns configured for this instance. Default checks are applied on columns dynamically. @@ -66,7 +64,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[List['DefaultColumnChecksPatternListModel']] + Response[List['ColumnQualityPolicyListModel']] """ kwargs = _get_kwargs() @@ -81,8 +79,8 @@ def sync_detailed( def sync( *, client: AuthenticatedClient, -) -> Optional[List["DefaultColumnChecksPatternListModel"]]: - """getAllDefaultColumnChecksPatterns +) -> Optional[List["ColumnQualityPolicyListModel"]]: + """getColumnQualityPolicies Returns a flat list of all column-level default check patterns configured for this instance. Default checks are applied on columns dynamically. @@ -92,7 +90,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - List['DefaultColumnChecksPatternListModel'] + List['ColumnQualityPolicyListModel'] """ return sync_detailed( @@ -103,8 +101,8 @@ def sync( async def asyncio_detailed( *, client: AuthenticatedClient, -) -> Response[List["DefaultColumnChecksPatternListModel"]]: - """getAllDefaultColumnChecksPatterns +) -> Response[List["ColumnQualityPolicyListModel"]]: + """getColumnQualityPolicies Returns a flat list of all column-level default check patterns configured for this instance. Default checks are applied on columns dynamically. @@ -114,7 +112,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[List['DefaultColumnChecksPatternListModel']] + Response[List['ColumnQualityPolicyListModel']] """ kwargs = _get_kwargs() @@ -127,8 +125,8 @@ async def asyncio_detailed( async def asyncio( *, client: AuthenticatedClient, -) -> Optional[List["DefaultColumnChecksPatternListModel"]]: - """getAllDefaultColumnChecksPatterns +) -> Optional[List["ColumnQualityPolicyListModel"]]: + """getColumnQualityPolicies Returns a flat list of all column-level default check patterns configured for this instance. Default checks are applied on columns dynamically. @@ -138,7 +136,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - List['DefaultColumnChecksPatternListModel'] + List['ColumnQualityPolicyListModel'] """ return ( diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_table_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy.py similarity index 80% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_default_table_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy.py index 8188d96058..7e8755cb37 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy.py @@ -5,7 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_table_checks_pattern_model import DefaultTableChecksPatternModel +from ...models.column_quality_policy_model import ColumnQualityPolicyModel from ...types import Response @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/table/{patternName}".format( + "url": "api/policies/checks/column/{patternName}".format( patternName=pattern_name, ), } @@ -25,9 +25,9 @@ def _get_kwargs( def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[DefaultTableChecksPatternModel]: +) -> Optional[ColumnQualityPolicyModel]: if response.status_code == HTTPStatus.OK: - response_200 = DefaultTableChecksPatternModel.from_dict(response.json()) + response_200 = ColumnQualityPolicyModel.from_dict(response.json()) return response_200 if client.raise_on_unexpected_status: @@ -38,7 +38,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[DefaultTableChecksPatternModel]: +) -> Response[ColumnQualityPolicyModel]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -51,8 +51,8 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultTableChecksPatternModel]: - """getDefaultTableChecksPattern +) -> Response[ColumnQualityPolicyModel]: + """getColumnQualityPolicy Returns a default checks pattern definition as a full specification object @@ -64,7 +64,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultTableChecksPatternModel] + Response[ColumnQualityPolicyModel] """ kwargs = _get_kwargs( @@ -82,8 +82,8 @@ def sync( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultTableChecksPatternModel]: - """getDefaultTableChecksPattern +) -> Optional[ColumnQualityPolicyModel]: + """getColumnQualityPolicy Returns a default checks pattern definition as a full specification object @@ -95,7 +95,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultTableChecksPatternModel + ColumnQualityPolicyModel """ return sync_detailed( @@ -108,8 +108,8 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultTableChecksPatternModel]: - """getDefaultTableChecksPattern +) -> Response[ColumnQualityPolicyModel]: + """getColumnQualityPolicy Returns a default checks pattern definition as a full specification object @@ -121,7 +121,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultTableChecksPatternModel] + Response[ColumnQualityPolicyModel] """ kwargs = _get_kwargs( @@ -137,8 +137,8 @@ async def asyncio( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultTableChecksPatternModel]: - """getDefaultTableChecksPattern +) -> Optional[ColumnQualityPolicyModel]: + """getColumnQualityPolicy Returns a default checks pattern definition as a full specification object @@ -150,7 +150,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultTableChecksPatternModel + ColumnQualityPolicyModel """ return ( diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_table_checks_pattern_target.py b/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy_target.py similarity index 78% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_default_table_checks_pattern_target.py rename to distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy_target.py index fc4745e67e..4a517c06f1 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_table_checks_pattern_target.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy_target.py @@ -5,9 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_table_checks_pattern_list_model import ( - DefaultTableChecksPatternListModel, -) +from ...models.column_quality_policy_list_model import ColumnQualityPolicyListModel from ...types import Response @@ -19,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/table/{patternName}/target".format( + "url": "api/policies/checks/column/{patternName}/target".format( patternName=pattern_name, ), } @@ -27,9 +25,9 @@ def _get_kwargs( def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[DefaultTableChecksPatternListModel]: +) -> Optional[ColumnQualityPolicyListModel]: if response.status_code == HTTPStatus.OK: - response_200 = DefaultTableChecksPatternListModel.from_dict(response.json()) + response_200 = ColumnQualityPolicyListModel.from_dict(response.json()) return response_200 if client.raise_on_unexpected_status: @@ -40,7 +38,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[DefaultTableChecksPatternListModel]: +) -> Response[ColumnQualityPolicyListModel]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -53,8 +51,8 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultTableChecksPatternListModel]: - """getDefaultTableChecksPatternTarget +) -> Response[ColumnQualityPolicyListModel]: + """getColumnQualityPolicyTarget Returns a default checks pattern definition @@ -66,7 +64,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultTableChecksPatternListModel] + Response[ColumnQualityPolicyListModel] """ kwargs = _get_kwargs( @@ -84,8 +82,8 @@ def sync( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultTableChecksPatternListModel]: - """getDefaultTableChecksPatternTarget +) -> Optional[ColumnQualityPolicyListModel]: + """getColumnQualityPolicyTarget Returns a default checks pattern definition @@ -97,7 +95,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultTableChecksPatternListModel + ColumnQualityPolicyListModel """ return sync_detailed( @@ -110,8 +108,8 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultTableChecksPatternListModel]: - """getDefaultTableChecksPatternTarget +) -> Response[ColumnQualityPolicyListModel]: + """getColumnQualityPolicyTarget Returns a default checks pattern definition @@ -123,7 +121,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultTableChecksPatternListModel] + Response[ColumnQualityPolicyListModel] """ kwargs = _get_kwargs( @@ -139,8 +137,8 @@ async def asyncio( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultTableChecksPatternListModel]: - """getDefaultTableChecksPatternTarget +) -> Optional[ColumnQualityPolicyListModel]: + """getColumnQualityPolicyTarget Returns a default checks pattern definition @@ -152,7 +150,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultTableChecksPatternListModel + ColumnQualityPolicyListModel """ return ( diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_monitoring_daily_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/get_monitoring_daily_column_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_default_monitoring_daily_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/get_monitoring_daily_column_quality_policy.py index fbd9885b4e..bf1f6c886f 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_monitoring_daily_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_monitoring_daily_column_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/column/{patternName}/monitoring/daily".format( + "url": "api/policies/checks/column/{patternName}/monitoring/daily".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringDailyColumnChecksPattern + """getMonitoringDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a column level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringDailyColumnChecksPattern + """getMonitoringDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a column level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringDailyColumnChecksPattern + """getMonitoringDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a column level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringDailyColumnChecksPattern + """getMonitoringDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a column level. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_monitoring_monthly_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/get_monitoring_monthly_column_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_default_monitoring_monthly_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/get_monitoring_monthly_column_quality_policy.py index f74a2d024e..21c2c62cfc 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_monitoring_monthly_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_monitoring_monthly_column_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/column/{patternName}/monitoring/monthly".format( + "url": "api/policies/checks/column/{patternName}/monitoring/monthly".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringMonthlyColumnChecksPattern + """getMonitoringMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a column level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringMonthlyColumnChecksPattern + """getMonitoringMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a column level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringMonthlyColumnChecksPattern + """getMonitoringMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a column level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringMonthlyColumnChecksPattern + """getMonitoringMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a column level. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_partitioned_daily_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/get_partitioned_daily_column_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_default_partitioned_daily_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/get_partitioned_daily_column_quality_policy.py index 30d293b5b1..95c9e49cf0 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_partitioned_daily_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_partitioned_daily_column_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/column/{patternName}/partitioned/daily".format( + "url": "api/policies/checks/column/{patternName}/partitioned/daily".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedDailyColumnChecksPattern + """getPartitionedDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a column level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedDailyColumnChecksPattern + """getPartitionedDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a column level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedDailyColumnChecksPattern + """getPartitionedDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a column level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedDailyColumnChecksPattern + """getPartitionedDailyColumnQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a column level. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_partitioned_monthly_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/get_partitioned_monthly_column_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_default_partitioned_monthly_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/get_partitioned_monthly_column_quality_policy.py index 450f3b4ac6..70def93cca 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_partitioned_monthly_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_partitioned_monthly_column_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/column/{patternName}/partitioned/monthly".format( + "url": "api/policies/checks/column/{patternName}/partitioned/monthly".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedMonthlyColumnChecksPattern + """getPartitionedMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a column level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedMonthlyColumnChecksPattern + """getPartitionedMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a column level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedMonthlyColumnChecksPattern + """getPartitionedMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a column level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedMonthlyColumnChecksPattern + """getPartitionedMonthlyColumnQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a column level. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_profiling_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/get_profiling_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_default_profiling_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/get_profiling_column_quality_policy.py index c83a874a73..ffea87c26b 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_profiling_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/get_profiling_column_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/column/{patternName}/profiling".format( + "url": "api/policies/checks/column/{patternName}/profiling".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultProfilingColumnChecksPattern + """getProfilingColumnQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a column level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultProfilingColumnChecksPattern + """getProfilingColumnQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a column level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultProfilingColumnChecksPattern + """getProfilingColumnQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a column level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultProfilingColumnChecksPattern + """getProfilingColumnQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a column level. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy.py similarity index 81% rename from distribution/python/dqops/client/api/default_column_check_patterns/update_default_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy.py index 0f0a093f74..b752d4015c 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy.py @@ -5,16 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_column_checks_pattern_model import ( - DefaultColumnChecksPatternModel, -) +from ...models.column_quality_policy_model import ColumnQualityPolicyModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultColumnChecksPatternModel, + json_body: ColumnQualityPolicyModel, ) -> Dict[str, Any]: pass @@ -23,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/column/{patternName}".format( + "url": "api/policies/checks/column/{patternName}".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,15 +54,16 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternModel, + json_body: ColumnQualityPolicyModel, ) -> Response[Any]: - """updateDefaultColumnChecksPattern + """updateColumnQualityPolicy Updates an default column-level checks pattern by saving a full specification object Args: pattern_name (str): - json_body (DefaultColumnChecksPatternModel): Default column-level checks pattern model + json_body (ColumnQualityPolicyModel): Default column-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -90,15 +89,16 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternModel, + json_body: ColumnQualityPolicyModel, ) -> Response[Any]: - """updateDefaultColumnChecksPattern + """updateColumnQualityPolicy Updates an default column-level checks pattern by saving a full specification object Args: pattern_name (str): - json_body (DefaultColumnChecksPatternModel): Default column-level checks pattern model + json_body (ColumnQualityPolicyModel): Default column-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_column_checks_pattern_target.py b/distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy_target.py similarity index 79% rename from distribution/python/dqops/client/api/default_column_check_patterns/update_default_column_checks_pattern_target.py rename to distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy_target.py index bfaca589f1..dde7db0c08 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_column_checks_pattern_target.py +++ b/distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy_target.py @@ -5,16 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_column_checks_pattern_list_model import ( - DefaultColumnChecksPatternListModel, -) +from ...models.column_quality_policy_list_model import ColumnQualityPolicyListModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultColumnChecksPatternListModel, + json_body: ColumnQualityPolicyListModel, ) -> Dict[str, Any]: pass @@ -23,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/column/{patternName}/target".format( + "url": "api/policies/checks/column/{patternName}/target".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,16 +54,16 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternListModel, + json_body: ColumnQualityPolicyListModel, ) -> Response[Any]: - """updateDefaultColumnChecksPatternTarget + """updateColumnQualityPolicyTarget Updates an default column-level checks pattern, changing only the target object Args: pattern_name (str): - json_body (DefaultColumnChecksPatternListModel): Default column-level checks pattern list - model + json_body (ColumnQualityPolicyListModel): Default column-level checks pattern (data + quality policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -91,16 +89,16 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultColumnChecksPatternListModel, + json_body: ColumnQualityPolicyListModel, ) -> Response[Any]: - """updateDefaultColumnChecksPatternTarget + """updateColumnQualityPolicyTarget Updates an default column-level checks pattern, changing only the target object Args: pattern_name (str): - json_body (DefaultColumnChecksPatternListModel): Default column-level checks pattern list - model + json_body (ColumnQualityPolicyListModel): Default column-level checks pattern (data + quality policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_monitoring_daily_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/update_monitoring_daily_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/update_default_monitoring_daily_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/update_monitoring_daily_column_quality_policy.py index 43a76e177d..8c017ec733 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_monitoring_daily_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/update_monitoring_daily_column_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/column/{patternName}/monitoring/daily".format( + "url": "api/policies/checks/column/{patternName}/monitoring/daily".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringDailyColumnChecksPattern + """updateMonitoringDailyColumnQualityPolicy New configuration of the default daily monitoring checks on a column level. These checks will be applied to columns. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringDailyColumnChecksPattern + """updateMonitoringDailyColumnQualityPolicy New configuration of the default daily monitoring checks on a column level. These checks will be applied to columns. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_monitoring_monthly_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/update_monitoring_monthly_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/update_default_monitoring_monthly_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/update_monitoring_monthly_column_quality_policy.py index 628518c319..e5c1f28649 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_monitoring_monthly_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/update_monitoring_monthly_column_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/column/{patternName}/monitoring/monthly".format( + "url": "api/policies/checks/column/{patternName}/monitoring/monthly".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringMonthlyColumnChecksPattern + """updateMonitoringMonthlyColumnQualityPolicy New configuration of the default monthly monitoring checks on a column level. These checks will be applied to columns. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringMonthlyColumnChecksPattern + """updateMonitoringMonthlyColumnQualityPolicy New configuration of the default monthly monitoring checks on a column level. These checks will be applied to columns. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_partitioned_daily_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/update_partitioned_daily_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/update_default_partitioned_daily_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/update_partitioned_daily_column_quality_policy.py index b546c456d1..5a70d1f31e 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_partitioned_daily_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/update_partitioned_daily_column_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/column/{patternName}/partitioned/daily".format( + "url": "api/policies/checks/column/{patternName}/partitioned/daily".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedDailyColumnChecksPattern + """updatePartitionedDailyColumnQualityPolicy New configuration of the default daily partitioned checks on a column level. These checks will be applied to columns. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedDailyColumnChecksPattern + """updatePartitionedDailyColumnQualityPolicy New configuration of the default daily partitioned checks on a column level. These checks will be applied to columns. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_partitioned_monthly_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/update_partitioned_monthly_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/update_default_partitioned_monthly_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/update_partitioned_monthly_column_quality_policy.py index f34b89e356..879eabcdb6 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_partitioned_monthly_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/update_partitioned_monthly_column_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/column/{patternName}/partitioned/monthly".format( + "url": "api/policies/checks/column/{patternName}/partitioned/monthly".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedMonthlyColumnChecksPattern + """updatePartitionedMonthlyColumnQualityPolicy New configuration of the default monthly partitioned checks on a column level. These checks will be applied to columns. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedMonthlyColumnChecksPattern + """updatePartitionedMonthlyColumnQualityPolicy New configuration of the default monthly partitioned checks on a column level. These checks will be applied to columns. diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_profiling_column_checks_pattern.py b/distribution/python/dqops/client/api/column_quality_policies/update_profiling_column_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_column_check_patterns/update_default_profiling_column_checks_pattern.py rename to distribution/python/dqops/client/api/column_quality_policies/update_profiling_column_quality_policy.py index 43748586ee..46b5b23654 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/update_default_profiling_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/column_quality_policies/update_profiling_column_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/column/{patternName}/profiling".format( + "url": "api/policies/checks/column/{patternName}/profiling".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultProfilingColumnChecksPattern + """updateProfilingColumnQualityPolicy New configuration of the default profiling checks on a column level. These checks will be applied to columns. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultProfilingColumnChecksPattern + """updateProfilingColumnQualityPolicy New configuration of the default profiling checks on a column level. These checks will be applied to columns. diff --git a/distribution/python/dqops/client/api/connections/create_connection_basic.py b/distribution/python/dqops/client/api/connections/create_connection_basic.py index 229c45e10d..54cdb5b473 100644 --- a/distribution/python/dqops/client/api/connections/create_connection_basic.py +++ b/distribution/python/dqops/client/api/connections/create_connection_basic.py @@ -62,8 +62,8 @@ def sync_detailed( Args: connection_name (str): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -97,8 +97,8 @@ async def asyncio_detailed( Args: connection_name (str): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/connections/update_connection_basic.py b/distribution/python/dqops/client/api/connections/update_connection_basic.py index d0982174c0..45fbe491b8 100644 --- a/distribution/python/dqops/client/api/connections/update_connection_basic.py +++ b/distribution/python/dqops/client/api/connections/update_connection_basic.py @@ -62,8 +62,8 @@ def sync_detailed( Args: connection_name (str): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -97,8 +97,8 @@ async def asyncio_detailed( Args: connection_name (str): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/__init__.py b/distribution/python/dqops/client/api/data_domains/__init__.py similarity index 100% rename from distribution/python/dqops/client/api/default_table_check_patterns/__init__.py rename to distribution/python/dqops/client/api/data_domains/__init__.py diff --git a/distribution/python/dqops/client/api/data_domains/create_data_domain.py b/distribution/python/dqops/client/api/data_domains/create_data_domain.py new file mode 100644 index 0000000000..46fbe6712f --- /dev/null +++ b/distribution/python/dqops/client/api/data_domains/create_data_domain.py @@ -0,0 +1,135 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.local_data_domain_model import LocalDataDomainModel +from ...types import Response + + +def _get_kwargs() -> Dict[str, Any]: + + pass + + return { + "method": "post", + "url": "api/domains/", + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[LocalDataDomainModel]: + if response.status_code == HTTPStatus.OK: + response_200 = LocalDataDomainModel.from_dict(response.json()) + + return response_200 + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[LocalDataDomainModel]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + *, + client: AuthenticatedClient, +) -> Response[LocalDataDomainModel]: + """createDataDomain + + Creates a new data domain given a data domain display name. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[LocalDataDomainModel] + """ + + kwargs = _get_kwargs() + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + *, + client: AuthenticatedClient, +) -> Optional[LocalDataDomainModel]: + """createDataDomain + + Creates a new data domain given a data domain display name. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + LocalDataDomainModel + """ + + return sync_detailed( + client=client, + ).parsed + + +async def asyncio_detailed( + *, + client: AuthenticatedClient, +) -> Response[LocalDataDomainModel]: + """createDataDomain + + Creates a new data domain given a data domain display name. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[LocalDataDomainModel] + """ + + kwargs = _get_kwargs() + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + *, + client: AuthenticatedClient, +) -> Optional[LocalDataDomainModel]: + """createDataDomain + + Creates a new data domain given a data domain display name. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + LocalDataDomainModel + """ + + return ( + await asyncio_detailed( + client=client, + ) + ).parsed diff --git a/distribution/python/dqops/client/api/data_domains/delete_data_domain.py b/distribution/python/dqops/client/api/data_domains/delete_data_domain.py new file mode 100644 index 0000000000..056b09a0ae --- /dev/null +++ b/distribution/python/dqops/client/api/data_domains/delete_data_domain.py @@ -0,0 +1,104 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...types import Response + + +def _get_kwargs( + data_domain_name: str, +) -> Dict[str, Any]: + + pass + + return { + "method": "delete", + "url": "api/domains/{dataDomainName}".format( + dataDomainName=data_domain_name, + ), + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[Any]: + if response.status_code == HTTPStatus.NO_CONTENT: + return None + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[Any]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + data_domain_name: str, + *, + client: AuthenticatedClient, +) -> Response[Any]: + """deleteDataDomain + + Deletes a data domain. The domain is deleted in the DQOps SaaS cloud and locally. + + Args: + data_domain_name (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + data_domain_name=data_domain_name, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +async def asyncio_detailed( + data_domain_name: str, + *, + client: AuthenticatedClient, +) -> Response[Any]: + """deleteDataDomain + + Deletes a data domain. The domain is deleted in the DQOps SaaS cloud and locally. + + Args: + data_domain_name (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + data_domain_name=data_domain_name, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) diff --git a/distribution/python/dqops/client/api/data_domains/get_local_data_domains.py b/distribution/python/dqops/client/api/data_domains/get_local_data_domains.py new file mode 100644 index 0000000000..d0a63a7563 --- /dev/null +++ b/distribution/python/dqops/client/api/data_domains/get_local_data_domains.py @@ -0,0 +1,144 @@ +from http import HTTPStatus +from typing import Any, Dict, List, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.local_data_domain_model import LocalDataDomainModel +from ...types import Response + + +def _get_kwargs() -> Dict[str, Any]: + + pass + + return { + "method": "get", + "url": "api/domains/", + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[List["LocalDataDomainModel"]]: + if response.status_code == HTTPStatus.OK: + response_200 = [] + _response_200 = response.json() + for response_200_item_data in _response_200: + response_200_item = LocalDataDomainModel.from_dict(response_200_item_data) + + response_200.append(response_200_item) + + return response_200 + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[List["LocalDataDomainModel"]]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + *, + client: AuthenticatedClient, +) -> Response[List["LocalDataDomainModel"]]: + """getLocalDataDomains + + Returns a list of local data domains that this instance is maintaining. Data domains are supported + only in an ENTERPRISE versions of DQOps. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[List['LocalDataDomainModel']] + """ + + kwargs = _get_kwargs() + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + *, + client: AuthenticatedClient, +) -> Optional[List["LocalDataDomainModel"]]: + """getLocalDataDomains + + Returns a list of local data domains that this instance is maintaining. Data domains are supported + only in an ENTERPRISE versions of DQOps. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + List['LocalDataDomainModel'] + """ + + return sync_detailed( + client=client, + ).parsed + + +async def asyncio_detailed( + *, + client: AuthenticatedClient, +) -> Response[List["LocalDataDomainModel"]]: + """getLocalDataDomains + + Returns a list of local data domains that this instance is maintaining. Data domains are supported + only in an ENTERPRISE versions of DQOps. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[List['LocalDataDomainModel']] + """ + + kwargs = _get_kwargs() + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + *, + client: AuthenticatedClient, +) -> Optional[List["LocalDataDomainModel"]]: + """getLocalDataDomains + + Returns a list of local data domains that this instance is maintaining. Data domains are supported + only in an ENTERPRISE versions of DQOps. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + List['LocalDataDomainModel'] + """ + + return ( + await asyncio_detailed( + client=client, + ) + ).parsed diff --git a/distribution/python/dqops/client/api/data_domains/switch_to_data_domain.py b/distribution/python/dqops/client/api/data_domains/switch_to_data_domain.py new file mode 100644 index 0000000000..389f206e15 --- /dev/null +++ b/distribution/python/dqops/client/api/data_domains/switch_to_data_domain.py @@ -0,0 +1,106 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...types import Response + + +def _get_kwargs( + data_domain_name: str, +) -> Dict[str, Any]: + + pass + + return { + "method": "get", + "url": "api/domains/{dataDomainName}/switch".format( + dataDomainName=data_domain_name, + ), + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[Any]: + if response.status_code == HTTPStatus.SEE_OTHER: + return None + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[Any]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + data_domain_name: str, + *, + client: AuthenticatedClient, +) -> Response[Any]: + """switchToDataDomain + + Switches to a different data domain. This operation sends a special cookie and redirects the user to + the home screen. + + Args: + data_domain_name (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + data_domain_name=data_domain_name, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +async def asyncio_detailed( + data_domain_name: str, + *, + client: AuthenticatedClient, +) -> Response[Any]: + """switchToDataDomain + + Switches to a different data domain. This operation sends a special cookie and redirects the user to + the home screen. + + Args: + data_domain_name (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + data_domain_name=data_domain_name, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) diff --git a/distribution/python/dqops/client/api/data_domains/synchronize_data_domains.py b/distribution/python/dqops/client/api/data_domains/synchronize_data_domains.py new file mode 100644 index 0000000000..5e59ee6300 --- /dev/null +++ b/distribution/python/dqops/client/api/data_domains/synchronize_data_domains.py @@ -0,0 +1,90 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...types import Response + + +def _get_kwargs() -> Dict[str, Any]: + + pass + + return { + "method": "patch", + "url": "api/domains/", + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[Any]: + if response.status_code == HTTPStatus.NO_CONTENT: + return None + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[Any]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + *, + client: AuthenticatedClient, +) -> Response[Any]: + """synchronizeDataDomains + + Synchronizes the domains in the SaaS cloud to this instance. All data domains will be created + locally. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs() + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +async def asyncio_detailed( + *, + client: AuthenticatedClient, +) -> Response[Any]: + """synchronizeDataDomains + + Synchronizes the domains in the SaaS cloud to this instance. All data domains will be created + locally. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs() + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) diff --git a/distribution/python/dqops/client/api/data_lineage/__init__.py b/distribution/python/dqops/client/api/data_lineage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/distribution/python/dqops/client/api/data_lineage/create_table_source_table.py b/distribution/python/dqops/client/api/data_lineage/create_table_source_table.py new file mode 100644 index 0000000000..417f943c18 --- /dev/null +++ b/distribution/python/dqops/client/api/data_lineage/create_table_source_table.py @@ -0,0 +1,156 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.table_lineage_source_spec import TableLineageSourceSpec +from ...types import Response + + +def _get_kwargs( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + json_body: TableLineageSourceSpec, +) -> Dict[str, Any]: + + pass + + json_json_body = json_body.to_dict() + + return { + "method": "post", + "url": "api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable}".format( + connectionName=connection_name, + schemaName=schema_name, + tableName=table_name, + sourceConnection=source_connection, + sourceSchema=source_schema, + sourceTable=source_table, + ), + "json": json_json_body, + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[Any]: + if response.status_code == HTTPStatus.CREATED: + return None + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[Any]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, + json_body: TableLineageSourceSpec, +) -> Response[Any]: + """createTableSourceTable + + Creates a new source table of the table's data lineage. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + json_body (TableLineageSourceSpec): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + json_body=json_body, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +async def asyncio_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, + json_body: TableLineageSourceSpec, +) -> Response[Any]: + """createTableSourceTable + + Creates a new source table of the table's data lineage. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + json_body (TableLineageSourceSpec): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + json_body=json_body, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) diff --git a/distribution/python/dqops/client/api/data_lineage/delete_table_source_table.py b/distribution/python/dqops/client/api/data_lineage/delete_table_source_table.py new file mode 100644 index 0000000000..8f912b04e4 --- /dev/null +++ b/distribution/python/dqops/client/api/data_lineage/delete_table_source_table.py @@ -0,0 +1,144 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...types import Response + + +def _get_kwargs( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, +) -> Dict[str, Any]: + + pass + + return { + "method": "delete", + "url": "api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable}".format( + connectionName=connection_name, + schemaName=schema_name, + tableName=table_name, + sourceConnection=source_connection, + sourceSchema=source_schema, + sourceTable=source_table, + ), + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[Any]: + if response.status_code == HTTPStatus.NO_CONTENT: + return None + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[Any]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, +) -> Response[Any]: + """deleteTableSourceTable + + Deletes a specific data lineage source table of the given table. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +async def asyncio_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, +) -> Response[Any]: + """deleteTableSourceTable + + Deletes a specific data lineage source table of the given table. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) diff --git a/distribution/python/dqops/client/api/data_lineage/get_table_source_table.py b/distribution/python/dqops/client/api/data_lineage/get_table_source_table.py new file mode 100644 index 0000000000..47fce4f723 --- /dev/null +++ b/distribution/python/dqops/client/api/data_lineage/get_table_source_table.py @@ -0,0 +1,231 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.table_lineage_source_spec import TableLineageSourceSpec +from ...types import Response + + +def _get_kwargs( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, +) -> Dict[str, Any]: + + pass + + return { + "method": "get", + "url": "api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable}".format( + connectionName=connection_name, + schemaName=schema_name, + tableName=table_name, + sourceConnection=source_connection, + sourceSchema=source_schema, + sourceTable=source_table, + ), + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[TableLineageSourceSpec]: + if response.status_code == HTTPStatus.OK: + response_200 = TableLineageSourceSpec.from_dict(response.json()) + + return response_200 + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[TableLineageSourceSpec]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, +) -> Response[TableLineageSourceSpec]: + """getTableSourceTable + + Reads a specific data lineage source table defined on a target tale. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[TableLineageSourceSpec] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, +) -> Optional[TableLineageSourceSpec]: + """getTableSourceTable + + Reads a specific data lineage source table defined on a target tale. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + TableLineageSourceSpec + """ + + return sync_detailed( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + client=client, + ).parsed + + +async def asyncio_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, +) -> Response[TableLineageSourceSpec]: + """getTableSourceTable + + Reads a specific data lineage source table defined on a target tale. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[TableLineageSourceSpec] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, +) -> Optional[TableLineageSourceSpec]: + """getTableSourceTable + + Reads a specific data lineage source table defined on a target tale. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + TableLineageSourceSpec + """ + + return ( + await asyncio_detailed( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + client=client, + ) + ).parsed diff --git a/distribution/python/dqops/client/api/data_lineage/get_table_source_tables.py b/distribution/python/dqops/client/api/data_lineage/get_table_source_tables.py new file mode 100644 index 0000000000..16e58fbb48 --- /dev/null +++ b/distribution/python/dqops/client/api/data_lineage/get_table_source_tables.py @@ -0,0 +1,221 @@ +from http import HTTPStatus +from typing import Any, Dict, List, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.check_type import CheckType +from ...models.table_lineage_source_list_model import TableLineageSourceListModel +from ...types import UNSET, Response, Unset + + +def _get_kwargs( + connection_name: str, + schema_name: str, + table_name: str, + *, + check_type: Union[Unset, None, CheckType] = UNSET, +) -> Dict[str, Any]: + + pass + + params: Dict[str, Any] = {} + json_check_type: Union[Unset, None, str] = UNSET + if not isinstance(check_type, Unset): + json_check_type = check_type.value if check_type else None + + params["checkType"] = json_check_type + + params = {k: v for k, v in params.items() if v is not UNSET and v is not None} + + return { + "method": "get", + "url": "api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage".format( + connectionName=connection_name, + schemaName=schema_name, + tableName=table_name, + ), + "params": params, + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[List["TableLineageSourceListModel"]]: + if response.status_code == HTTPStatus.OK: + response_200 = [] + _response_200 = response.json() + for response_200_item_data in _response_200: + response_200_item = TableLineageSourceListModel.from_dict( + response_200_item_data + ) + + response_200.append(response_200_item) + + return response_200 + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[List["TableLineageSourceListModel"]]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + connection_name: str, + schema_name: str, + table_name: str, + *, + client: AuthenticatedClient, + check_type: Union[Unset, None, CheckType] = UNSET, +) -> Response[List["TableLineageSourceListModel"]]: + """getTableSourceTables + + Returns a list of source tables on the data lineage that are sources of the given table. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + check_type (Union[Unset, None, CheckType]): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[List['TableLineageSourceListModel']] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + check_type=check_type, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + connection_name: str, + schema_name: str, + table_name: str, + *, + client: AuthenticatedClient, + check_type: Union[Unset, None, CheckType] = UNSET, +) -> Optional[List["TableLineageSourceListModel"]]: + """getTableSourceTables + + Returns a list of source tables on the data lineage that are sources of the given table. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + check_type (Union[Unset, None, CheckType]): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + List['TableLineageSourceListModel'] + """ + + return sync_detailed( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + client=client, + check_type=check_type, + ).parsed + + +async def asyncio_detailed( + connection_name: str, + schema_name: str, + table_name: str, + *, + client: AuthenticatedClient, + check_type: Union[Unset, None, CheckType] = UNSET, +) -> Response[List["TableLineageSourceListModel"]]: + """getTableSourceTables + + Returns a list of source tables on the data lineage that are sources of the given table. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + check_type (Union[Unset, None, CheckType]): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[List['TableLineageSourceListModel']] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + check_type=check_type, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + connection_name: str, + schema_name: str, + table_name: str, + *, + client: AuthenticatedClient, + check_type: Union[Unset, None, CheckType] = UNSET, +) -> Optional[List["TableLineageSourceListModel"]]: + """getTableSourceTables + + Returns a list of source tables on the data lineage that are sources of the given table. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + check_type (Union[Unset, None, CheckType]): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + List['TableLineageSourceListModel'] + """ + + return ( + await asyncio_detailed( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + client=client, + check_type=check_type, + ) + ).parsed diff --git a/distribution/python/dqops/client/api/data_lineage/update_table_source_table.py b/distribution/python/dqops/client/api/data_lineage/update_table_source_table.py new file mode 100644 index 0000000000..c58830dfcf --- /dev/null +++ b/distribution/python/dqops/client/api/data_lineage/update_table_source_table.py @@ -0,0 +1,156 @@ +from http import HTTPStatus +from typing import Any, Dict, Optional, Union + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.table_lineage_source_spec import TableLineageSourceSpec +from ...types import Response + + +def _get_kwargs( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + json_body: TableLineageSourceSpec, +) -> Dict[str, Any]: + + pass + + json_json_body = json_body.to_dict() + + return { + "method": "put", + "url": "api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable}".format( + connectionName=connection_name, + schemaName=schema_name, + tableName=table_name, + sourceConnection=source_connection, + sourceSchema=source_schema, + sourceTable=source_table, + ), + "json": json_json_body, + } + + +def _parse_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Optional[Any]: + if response.status_code == HTTPStatus.NO_CONTENT: + return None + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: Union[AuthenticatedClient, Client], response: httpx.Response +) -> Response[Any]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, + json_body: TableLineageSourceSpec, +) -> Response[Any]: + """updateTableSourceTable + + Update a specific data lineage source table using a new model. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + json_body (TableLineageSourceSpec): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + json_body=json_body, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +async def asyncio_detailed( + connection_name: str, + schema_name: str, + table_name: str, + source_connection: str, + source_schema: str, + source_table: str, + *, + client: AuthenticatedClient, + json_body: TableLineageSourceSpec, +) -> Response[Any]: + """updateTableSourceTable + + Update a specific data lineage source table using a new model. + + Args: + connection_name (str): + schema_name (str): + table_name (str): + source_connection (str): + source_schema (str): + source_table (str): + json_body (TableLineageSourceSpec): + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[Any] + """ + + kwargs = _get_kwargs( + connection_name=connection_name, + schema_name=schema_name, + table_name=table_name, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + json_body=json_body, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) diff --git a/distribution/python/dqops/client/api/data_sources/test_connection.py b/distribution/python/dqops/client/api/data_sources/test_connection.py index ec3e8ec8a0..cc26c03673 100644 --- a/distribution/python/dqops/client/api/data_sources/test_connection.py +++ b/distribution/python/dqops/client/api/data_sources/test_connection.py @@ -69,8 +69,8 @@ def sync_detailed( Args: verify_name_uniqueness (Union[Unset, None, bool]): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -104,8 +104,8 @@ def sync( Args: verify_name_uniqueness (Union[Unset, None, bool]): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -134,8 +134,8 @@ async def asyncio_detailed( Args: verify_name_uniqueness (Union[Unset, None, bool]): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -167,8 +167,8 @@ async def asyncio( Args: verify_name_uniqueness (Union[Unset, None, bool]): - json_body (ConnectionModel): Connection model for with a subset of parameters, excluding - all nested objects. + json_body (ConnectionModel): Connection model with a subset of parameters, excluding all + nested objects. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/jobs/collect_statistics_on_data_groups.py b/distribution/python/dqops/client/api/jobs/collect_statistics_on_data_groups.py index c3bc07a0f4..25a13efc4a 100644 --- a/distribution/python/dqops/client/api/jobs/collect_statistics_on_data_groups.py +++ b/distribution/python/dqops/client/api/jobs/collect_statistics_on_data_groups.py @@ -18,6 +18,8 @@ def _get_kwargs( *, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Dict[str, Any]: @@ -27,6 +29,10 @@ def _get_kwargs( params: Dict[str, Any] = {} params["jobBusinessKey"] = job_business_key + params["samplesLimit"] = samples_limit + + params["configureTable"] = configure_table + params["wait"] = wait params["waitTimeout"] = wait_timeout @@ -72,6 +78,8 @@ def sync_detailed( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Response[CollectStatisticsQueueJobResult]: @@ -82,6 +90,8 @@ def sync_detailed( Args: job_business_key (Union[Unset, None, str]): + samples_limit (Union[Unset, None, int]): + configure_table (Union[Unset, None, bool]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -97,6 +107,8 @@ def sync_detailed( kwargs = _get_kwargs( json_body=json_body, job_business_key=job_business_key, + samples_limit=samples_limit, + configure_table=configure_table, wait=wait, wait_timeout=wait_timeout, ) @@ -113,6 +125,8 @@ def sync( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Optional[CollectStatisticsQueueJobResult]: @@ -123,6 +137,8 @@ def sync( Args: job_business_key (Union[Unset, None, str]): + samples_limit (Union[Unset, None, int]): + configure_table (Union[Unset, None, bool]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -139,6 +155,8 @@ def sync( client=client, json_body=json_body, job_business_key=job_business_key, + samples_limit=samples_limit, + configure_table=configure_table, wait=wait, wait_timeout=wait_timeout, ).parsed @@ -149,6 +167,8 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Response[CollectStatisticsQueueJobResult]: @@ -159,6 +179,8 @@ async def asyncio_detailed( Args: job_business_key (Union[Unset, None, str]): + samples_limit (Union[Unset, None, int]): + configure_table (Union[Unset, None, bool]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -174,6 +196,8 @@ async def asyncio_detailed( kwargs = _get_kwargs( json_body=json_body, job_business_key=job_business_key, + samples_limit=samples_limit, + configure_table=configure_table, wait=wait, wait_timeout=wait_timeout, ) @@ -188,6 +212,8 @@ async def asyncio( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Optional[CollectStatisticsQueueJobResult]: @@ -198,6 +224,8 @@ async def asyncio( Args: job_business_key (Union[Unset, None, str]): + samples_limit (Union[Unset, None, int]): + configure_table (Union[Unset, None, bool]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -215,6 +243,8 @@ async def asyncio( client=client, json_body=json_body, job_business_key=job_business_key, + samples_limit=samples_limit, + configure_table=configure_table, wait=wait, wait_timeout=wait_timeout, ) diff --git a/distribution/python/dqops/client/api/jobs/collect_statistics_on_table.py b/distribution/python/dqops/client/api/jobs/collect_statistics_on_table.py index 8a78817e48..ae52876522 100644 --- a/distribution/python/dqops/client/api/jobs/collect_statistics_on_table.py +++ b/distribution/python/dqops/client/api/jobs/collect_statistics_on_table.py @@ -18,6 +18,8 @@ def _get_kwargs( *, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Dict[str, Any]: @@ -27,6 +29,10 @@ def _get_kwargs( params: Dict[str, Any] = {} params["jobBusinessKey"] = job_business_key + params["configureTable"] = configure_table + + params["samplesLimit"] = samples_limit + params["wait"] = wait params["waitTimeout"] = wait_timeout @@ -72,6 +78,8 @@ def sync_detailed( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Response[CollectStatisticsQueueJobResult]: @@ -81,6 +89,8 @@ def sync_detailed( Args: job_business_key (Union[Unset, None, str]): + configure_table (Union[Unset, None, bool]): + samples_limit (Union[Unset, None, int]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -96,6 +106,8 @@ def sync_detailed( kwargs = _get_kwargs( json_body=json_body, job_business_key=job_business_key, + configure_table=configure_table, + samples_limit=samples_limit, wait=wait, wait_timeout=wait_timeout, ) @@ -112,6 +124,8 @@ def sync( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Optional[CollectStatisticsQueueJobResult]: @@ -121,6 +135,8 @@ def sync( Args: job_business_key (Union[Unset, None, str]): + configure_table (Union[Unset, None, bool]): + samples_limit (Union[Unset, None, int]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -137,6 +153,8 @@ def sync( client=client, json_body=json_body, job_business_key=job_business_key, + configure_table=configure_table, + samples_limit=samples_limit, wait=wait, wait_timeout=wait_timeout, ).parsed @@ -147,6 +165,8 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Response[CollectStatisticsQueueJobResult]: @@ -156,6 +176,8 @@ async def asyncio_detailed( Args: job_business_key (Union[Unset, None, str]): + configure_table (Union[Unset, None, bool]): + samples_limit (Union[Unset, None, int]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -171,6 +193,8 @@ async def asyncio_detailed( kwargs = _get_kwargs( json_body=json_body, job_business_key=job_business_key, + configure_table=configure_table, + samples_limit=samples_limit, wait=wait, wait_timeout=wait_timeout, ) @@ -185,6 +209,8 @@ async def asyncio( client: AuthenticatedClient, json_body: StatisticsCollectorSearchFilters, job_business_key: Union[Unset, None, str] = UNSET, + configure_table: Union[Unset, None, bool] = UNSET, + samples_limit: Union[Unset, None, int] = UNSET, wait: Union[Unset, None, bool] = UNSET, wait_timeout: Union[Unset, None, int] = UNSET, ) -> Optional[CollectStatisticsQueueJobResult]: @@ -194,6 +220,8 @@ async def asyncio( Args: job_business_key (Union[Unset, None, str]): + configure_table (Union[Unset, None, bool]): + samples_limit (Union[Unset, None, int]): wait (Union[Unset, None, bool]): wait_timeout (Union[Unset, None, int]): json_body (StatisticsCollectorSearchFilters): @@ -211,6 +239,8 @@ async def asyncio( client=client, json_body=json_body, job_business_key=job_business_key, + configure_table=configure_table, + samples_limit=samples_limit, wait=wait, wait_timeout=wait_timeout, ) diff --git a/distribution/python/dqops/client/api/table_quality_policies/__init__.py b/distribution/python/dqops/client/api/table_quality_policies/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/copy_from_default_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/copy_from_table_quality_policy.py similarity index 89% rename from distribution/python/dqops/client/api/default_table_check_patterns/copy_from_default_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/copy_from_table_quality_policy.py index 5f3dff0f3f..6e487715be 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/copy_from_default_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/copy_from_table_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "post", - "url": "api/default/checks/table/{targetPatternName}/copyfrom/{sourcePatternName}".format( + "url": "api/policies/checks/table/{targetPatternName}/copyfrom/{sourcePatternName}".format( targetPatternName=target_pattern_name, sourcePatternName=source_pattern_name, ), @@ -52,10 +52,10 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """copyFromDefaultTableChecksPattern + """copyFromTableQualityPolicy - Creates (adds) a copy of an existing default table-level checks pattern configuration, under a new - name. + Creates (adds) a copy of an existing default table-level checks pattern configuration (data quality + policy) under a new name. Args: target_pattern_name (str): @@ -87,10 +87,10 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """copyFromDefaultTableChecksPattern + """copyFromTableQualityPolicy - Creates (adds) a copy of an existing default table-level checks pattern configuration, under a new - name. + Creates (adds) a copy of an existing default table-level checks pattern configuration (data quality + policy) under a new name. Args: target_pattern_name (str): diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/create_default_table_checks_pattern_target.py b/distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_pattern.py similarity index 73% rename from distribution/python/dqops/client/api/default_table_check_patterns/create_default_table_checks_pattern_target.py rename to distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_pattern.py index 60651fe945..c8199db2d2 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/create_default_table_checks_pattern_target.py +++ b/distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_pattern.py @@ -5,16 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_table_checks_pattern_list_model import ( - DefaultTableChecksPatternListModel, -) +from ...models.table_quality_policy_model import TableQualityPolicyModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultTableChecksPatternListModel, + json_body: TableQualityPolicyModel, ) -> Dict[str, Any]: pass @@ -23,7 +21,7 @@ def _get_kwargs( return { "method": "post", - "url": "api/default/checks/table/{patternName}/target".format( + "url": "api/policies/checks/table/{patternName}".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,16 +54,17 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternListModel, + json_body: TableQualityPolicyModel, ) -> Response[Any]: - """createDefaultTableChecksPatternTarget + """createTableQualityPolicyPattern - Creates (adds) a new default table-level checks pattern configuration. + Creates (adds) a new default table-level checks pattern (data quality policy) configuration by + saving a full specification object. Args: pattern_name (str): - json_body (DefaultTableChecksPatternListModel): Default table-level checks pattern list - model + json_body (TableQualityPolicyModel): Default table-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -91,16 +90,17 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternListModel, + json_body: TableQualityPolicyModel, ) -> Response[Any]: - """createDefaultTableChecksPatternTarget + """createTableQualityPolicyPattern - Creates (adds) a new default table-level checks pattern configuration. + Creates (adds) a new default table-level checks pattern (data quality policy) configuration by + saving a full specification object. Args: pattern_name (str): - json_body (DefaultTableChecksPatternListModel): Default table-level checks pattern list - model + json_body (TableQualityPolicyModel): Default table-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/create_default_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_target.py similarity index 77% rename from distribution/python/dqops/client/api/default_table_check_patterns/create_default_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_target.py index f906bc5f51..f4bf2914f5 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/create_default_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_target.py @@ -5,14 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_table_checks_pattern_model import DefaultTableChecksPatternModel +from ...models.table_quality_policy_list_model import TableQualityPolicyListModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultTableChecksPatternModel, + json_body: TableQualityPolicyListModel, ) -> Dict[str, Any]: pass @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "post", - "url": "api/default/checks/table/{patternName}".format( + "url": "api/policies/checks/table/{patternName}/target".format( patternName=pattern_name, ), "json": json_json_body, @@ -54,16 +54,17 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternModel, + json_body: TableQualityPolicyListModel, ) -> Response[Any]: - """createDefaultTableChecksPattern + """createTableQualityPolicyTarget - Creates (adds) a new default table-level checks pattern configuration by saving a full specification - object. + Creates (adds) a new default table-level checks pattern configuration (a table-level data quality + policy). Args: pattern_name (str): - json_body (DefaultTableChecksPatternModel): Default table-level checks pattern model + json_body (TableQualityPolicyListModel): Default table-level checks pattern (data quality + policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -89,16 +90,17 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternModel, + json_body: TableQualityPolicyListModel, ) -> Response[Any]: - """createDefaultTableChecksPattern + """createTableQualityPolicyTarget - Creates (adds) a new default table-level checks pattern configuration by saving a full specification - object. + Creates (adds) a new default table-level checks pattern configuration (a table-level data quality + policy). Args: pattern_name (str): - json_body (DefaultTableChecksPatternModel): Default table-level checks pattern model + json_body (TableQualityPolicyListModel): Default table-level checks pattern (data quality + policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/delete_default_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/delete_table_quality_policy.py similarity index 87% rename from distribution/python/dqops/client/api/default_table_check_patterns/delete_default_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/delete_table_quality_policy.py index 225449b289..e15cd47f73 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/delete_default_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/delete_table_quality_policy.py @@ -16,7 +16,7 @@ def _get_kwargs( return { "method": "delete", - "url": "api/default/checks/table/{patternName}".format( + "url": "api/policies/checks/table/{patternName}".format( patternName=pattern_name, ), } @@ -49,9 +49,9 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """deleteDefaultTableChecksPattern + """deleteTableQualityPolicy - Deletes a default table-level checks pattern + Deletes a default table-level checks pattern (a data quality policy at a column level). Args: pattern_name (str): @@ -80,9 +80,9 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[Any]: - """deleteDefaultTableChecksPattern + """deleteTableQualityPolicy - Deletes a default table-level checks pattern + Deletes a default table-level checks pattern (a data quality policy at a column level). Args: pattern_name (str): diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_monitoring_daily_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/get_monitoring_daily_table_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_default_monitoring_daily_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/get_monitoring_daily_table_quality_policy.py index 21b0411d1f..7854006f60 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_monitoring_daily_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_monitoring_daily_table_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/table/{patternName}/monitoring/daily".format( + "url": "api/policies/checks/table/{patternName}/monitoring/daily".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringDailyTableChecksPattern + """getMonitoringDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a table level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringDailyTableChecksPattern + """getMonitoringDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a table level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringDailyTableChecksPattern + """getMonitoringDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a table level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringDailyTableChecksPattern + """getMonitoringDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a table level. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_monitoring_monthly_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/get_monitoring_monthly_table_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_default_monitoring_monthly_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/get_monitoring_monthly_table_quality_policy.py index dcf6f44657..880fddda44 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_monitoring_monthly_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_monitoring_monthly_table_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/table/{patternName}/monitoring/monthly".format( + "url": "api/policies/checks/table/{patternName}/monitoring/monthly".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringMonthlyTableChecksPattern + """getMonitoringMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a table level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringMonthlyTableChecksPattern + """getMonitoringMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a table level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultMonitoringMonthlyTableChecksPattern + """getMonitoringMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a table level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultMonitoringMonthlyTableChecksPattern + """getMonitoringMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a table level. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_partitioned_daily_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/get_partitioned_daily_table_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_default_partitioned_daily_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/get_partitioned_daily_table_quality_policy.py index 6b20f23df2..6580a3a0e1 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_partitioned_daily_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_partitioned_daily_table_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/table/{patternName}/partitioned/daily".format( + "url": "api/policies/checks/table/{patternName}/partitioned/daily".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedDailyTableChecksPattern + """getPartitionedDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a table level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedDailyTableChecksPattern + """getPartitionedDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a table level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedDailyTableChecksPattern + """getPartitionedDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a table level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedDailyTableChecksPattern + """getPartitionedDailyTableQualityPolicy Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a table level. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_partitioned_monthly_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/get_partitioned_monthly_table_quality_policy.py similarity index 93% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_default_partitioned_monthly_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/get_partitioned_monthly_table_quality_policy.py index 42de6f46c1..ad33606e5f 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_partitioned_monthly_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_partitioned_monthly_table_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/table/{patternName}/partitioned/monthly".format( + "url": "api/policies/checks/table/{patternName}/partitioned/monthly".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedMonthlyTableChecksPattern + """getPartitionedMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a table level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedMonthlyTableChecksPattern + """getPartitionedMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a table level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultPartitionedMonthlyTableChecksPattern + """getPartitionedMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a table level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultPartitionedMonthlyTableChecksPattern + """getPartitionedMonthlyTableQualityPolicy Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a table level. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_profiling_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/get_profiling_table_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_default_profiling_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/get_profiling_table_quality_policy.py index 520d48ca25..9888f5bef4 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_default_profiling_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_profiling_table_quality_policy.py @@ -17,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/table/{patternName}/profiling".format( + "url": "api/policies/checks/table/{patternName}/profiling".format( patternName=pattern_name, ), } @@ -52,7 +52,7 @@ def sync_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultProfilingTableChecksPattern + """getProfilingTableQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a table level. @@ -84,7 +84,7 @@ def sync( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultProfilingTableChecksPattern + """getProfilingTableQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a table level. @@ -111,7 +111,7 @@ async def asyncio_detailed( *, client: AuthenticatedClient, ) -> Response[CheckContainerModel]: - """getDefaultProfilingTableChecksPattern + """getProfilingTableQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a table level. @@ -141,7 +141,7 @@ async def asyncio( *, client: AuthenticatedClient, ) -> Optional[CheckContainerModel]: - """getDefaultProfilingTableChecksPattern + """getProfilingTableQualityPolicy Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a table level. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/get_all_default_table_checks_patterns.py b/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policies.py similarity index 70% rename from distribution/python/dqops/client/api/default_table_check_patterns/get_all_default_table_checks_patterns.py rename to distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policies.py index fdf4d628cb..39744188e8 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/get_all_default_table_checks_patterns.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policies.py @@ -5,9 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_table_checks_pattern_list_model import ( - DefaultTableChecksPatternListModel, -) +from ...models.table_quality_policy_list_model import TableQualityPolicyListModel from ...types import Response @@ -17,18 +15,18 @@ def _get_kwargs() -> Dict[str, Any]: return { "method": "get", - "url": "api/default/checks/table", + "url": "api/policies/checks/table", } def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[List["DefaultTableChecksPatternListModel"]]: +) -> Optional[List["TableQualityPolicyListModel"]]: if response.status_code == HTTPStatus.OK: response_200 = [] _response_200 = response.json() for response_200_item_data in _response_200: - response_200_item = DefaultTableChecksPatternListModel.from_dict( + response_200_item = TableQualityPolicyListModel.from_dict( response_200_item_data ) @@ -43,7 +41,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[List["DefaultTableChecksPatternListModel"]]: +) -> Response[List["TableQualityPolicyListModel"]]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -55,18 +53,18 @@ def _build_response( def sync_detailed( *, client: AuthenticatedClient, -) -> Response[List["DefaultTableChecksPatternListModel"]]: - """getAllDefaultTableChecksPatterns +) -> Response[List["TableQualityPolicyListModel"]]: + """getTableQualityPolicies - Returns a flat list of all table-level default check patterns configured for this instance. Default - checks are applied on tables dynamically. + Returns a flat list of all table-level default check patterns (data quality policies) configured for + this instance. Default checks are applied on tables dynamically. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[List['DefaultTableChecksPatternListModel']] + Response[List['TableQualityPolicyListModel']] """ kwargs = _get_kwargs() @@ -81,18 +79,18 @@ def sync_detailed( def sync( *, client: AuthenticatedClient, -) -> Optional[List["DefaultTableChecksPatternListModel"]]: - """getAllDefaultTableChecksPatterns +) -> Optional[List["TableQualityPolicyListModel"]]: + """getTableQualityPolicies - Returns a flat list of all table-level default check patterns configured for this instance. Default - checks are applied on tables dynamically. + Returns a flat list of all table-level default check patterns (data quality policies) configured for + this instance. Default checks are applied on tables dynamically. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - List['DefaultTableChecksPatternListModel'] + List['TableQualityPolicyListModel'] """ return sync_detailed( @@ -103,18 +101,18 @@ def sync( async def asyncio_detailed( *, client: AuthenticatedClient, -) -> Response[List["DefaultTableChecksPatternListModel"]]: - """getAllDefaultTableChecksPatterns +) -> Response[List["TableQualityPolicyListModel"]]: + """getTableQualityPolicies - Returns a flat list of all table-level default check patterns configured for this instance. Default - checks are applied on tables dynamically. + Returns a flat list of all table-level default check patterns (data quality policies) configured for + this instance. Default checks are applied on tables dynamically. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[List['DefaultTableChecksPatternListModel']] + Response[List['TableQualityPolicyListModel']] """ kwargs = _get_kwargs() @@ -127,18 +125,18 @@ async def asyncio_detailed( async def asyncio( *, client: AuthenticatedClient, -) -> Optional[List["DefaultTableChecksPatternListModel"]]: - """getAllDefaultTableChecksPatterns +) -> Optional[List["TableQualityPolicyListModel"]]: + """getTableQualityPolicies - Returns a flat list of all table-level default check patterns configured for this instance. Default - checks are applied on tables dynamically. + Returns a flat list of all table-level default check patterns (data quality policies) configured for + this instance. Default checks are applied on tables dynamically. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - List['DefaultTableChecksPatternListModel'] + List['TableQualityPolicyListModel'] """ return ( diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_column_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy.py similarity index 72% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_default_column_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy.py index 8235744b1f..c075642f79 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_column_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy.py @@ -5,9 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_column_checks_pattern_model import ( - DefaultColumnChecksPatternModel, -) +from ...models.table_quality_policy_model import TableQualityPolicyModel from ...types import Response @@ -19,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/column/{patternName}".format( + "url": "api/policies/checks/table/{patternName}".format( patternName=pattern_name, ), } @@ -27,9 +25,9 @@ def _get_kwargs( def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[DefaultColumnChecksPatternModel]: +) -> Optional[TableQualityPolicyModel]: if response.status_code == HTTPStatus.OK: - response_200 = DefaultColumnChecksPatternModel.from_dict(response.json()) + response_200 = TableQualityPolicyModel.from_dict(response.json()) return response_200 if client.raise_on_unexpected_status: @@ -40,7 +38,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[DefaultColumnChecksPatternModel]: +) -> Response[TableQualityPolicyModel]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -53,10 +51,11 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultColumnChecksPatternModel]: - """getDefaultColumnChecksPattern +) -> Response[TableQualityPolicyModel]: + """getTableQualityPolicy - Returns a default checks pattern definition as a full specification object + Returns a default table-level checks pattern (data quality policy) definition as a full + specification object Args: pattern_name (str): @@ -66,7 +65,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultColumnChecksPatternModel] + Response[TableQualityPolicyModel] """ kwargs = _get_kwargs( @@ -84,10 +83,11 @@ def sync( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultColumnChecksPatternModel]: - """getDefaultColumnChecksPattern +) -> Optional[TableQualityPolicyModel]: + """getTableQualityPolicy - Returns a default checks pattern definition as a full specification object + Returns a default table-level checks pattern (data quality policy) definition as a full + specification object Args: pattern_name (str): @@ -97,7 +97,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultColumnChecksPatternModel + TableQualityPolicyModel """ return sync_detailed( @@ -110,10 +110,11 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultColumnChecksPatternModel]: - """getDefaultColumnChecksPattern +) -> Response[TableQualityPolicyModel]: + """getTableQualityPolicy - Returns a default checks pattern definition as a full specification object + Returns a default table-level checks pattern (data quality policy) definition as a full + specification object Args: pattern_name (str): @@ -123,7 +124,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultColumnChecksPatternModel] + Response[TableQualityPolicyModel] """ kwargs = _get_kwargs( @@ -139,10 +140,11 @@ async def asyncio( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultColumnChecksPatternModel]: - """getDefaultColumnChecksPattern +) -> Optional[TableQualityPolicyModel]: + """getTableQualityPolicy - Returns a default checks pattern definition as a full specification object + Returns a default table-level checks pattern (data quality policy) definition as a full + specification object Args: pattern_name (str): @@ -152,7 +154,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultColumnChecksPatternModel + TableQualityPolicyModel """ return ( diff --git a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_column_checks_pattern_target.py b/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy_target.py similarity index 73% rename from distribution/python/dqops/client/api/default_column_check_patterns/get_default_column_checks_pattern_target.py rename to distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy_target.py index 3aa8d31d60..0ca565f9b3 100644 --- a/distribution/python/dqops/client/api/default_column_check_patterns/get_default_column_checks_pattern_target.py +++ b/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy_target.py @@ -5,9 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_column_checks_pattern_list_model import ( - DefaultColumnChecksPatternListModel, -) +from ...models.table_quality_policy_list_model import TableQualityPolicyListModel from ...types import Response @@ -19,7 +17,7 @@ def _get_kwargs( return { "method": "get", - "url": "api/default/checks/column/{patternName}/target".format( + "url": "api/policies/checks/table/{patternName}/target".format( patternName=pattern_name, ), } @@ -27,9 +25,9 @@ def _get_kwargs( def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[DefaultColumnChecksPatternListModel]: +) -> Optional[TableQualityPolicyListModel]: if response.status_code == HTTPStatus.OK: - response_200 = DefaultColumnChecksPatternListModel.from_dict(response.json()) + response_200 = TableQualityPolicyListModel.from_dict(response.json()) return response_200 if client.raise_on_unexpected_status: @@ -40,7 +38,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[DefaultColumnChecksPatternListModel]: +) -> Response[TableQualityPolicyListModel]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -53,10 +51,10 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultColumnChecksPatternListModel]: - """getDefaultColumnChecksPatternTarget +) -> Response[TableQualityPolicyListModel]: + """getTableQualityPolicyTarget - Returns a default checks pattern definition + Returns a default checks pattern definition (a data quality policy) Args: pattern_name (str): @@ -66,7 +64,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultColumnChecksPatternListModel] + Response[TableQualityPolicyListModel] """ kwargs = _get_kwargs( @@ -84,10 +82,10 @@ def sync( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultColumnChecksPatternListModel]: - """getDefaultColumnChecksPatternTarget +) -> Optional[TableQualityPolicyListModel]: + """getTableQualityPolicyTarget - Returns a default checks pattern definition + Returns a default checks pattern definition (a data quality policy) Args: pattern_name (str): @@ -97,7 +95,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultColumnChecksPatternListModel + TableQualityPolicyListModel """ return sync_detailed( @@ -110,10 +108,10 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, -) -> Response[DefaultColumnChecksPatternListModel]: - """getDefaultColumnChecksPatternTarget +) -> Response[TableQualityPolicyListModel]: + """getTableQualityPolicyTarget - Returns a default checks pattern definition + Returns a default checks pattern definition (a data quality policy) Args: pattern_name (str): @@ -123,7 +121,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DefaultColumnChecksPatternListModel] + Response[TableQualityPolicyListModel] """ kwargs = _get_kwargs( @@ -139,10 +137,10 @@ async def asyncio( pattern_name: str, *, client: AuthenticatedClient, -) -> Optional[DefaultColumnChecksPatternListModel]: - """getDefaultColumnChecksPatternTarget +) -> Optional[TableQualityPolicyListModel]: + """getTableQualityPolicyTarget - Returns a default checks pattern definition + Returns a default checks pattern definition (a data quality policy) Args: pattern_name (str): @@ -152,7 +150,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DefaultColumnChecksPatternListModel + TableQualityPolicyListModel """ return ( diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_monitoring_daily_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/update_monitoring_daily_table_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_table_check_patterns/update_default_monitoring_daily_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/update_monitoring_daily_table_quality_policy.py index b5a7ed4075..1100d96024 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_monitoring_daily_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/update_monitoring_daily_table_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/table/{patternName}/monitoring/daily".format( + "url": "api/policies/checks/table/{patternName}/monitoring/daily".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringDailyTableChecksPattern + """updateMonitoringDailyTableQualityPolicy New configuration of the default daily monitoring checks on a table level. These checks will be applied to tables. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringDailyTableChecksPattern + """updateMonitoringDailyTableQualityPolicy New configuration of the default daily monitoring checks on a table level. These checks will be applied to tables. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_monitoring_monthly_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/update_monitoring_monthly_table_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_table_check_patterns/update_default_monitoring_monthly_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/update_monitoring_monthly_table_quality_policy.py index 0cd0d346f4..6f63432e36 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_monitoring_monthly_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/update_monitoring_monthly_table_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/table/{patternName}/monitoring/monthly".format( + "url": "api/policies/checks/table/{patternName}/monitoring/monthly".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringMonthlyTableChecksPattern + """updateMonitoringMonthlyTableQualityPolicy New configuration of the default monthly monitoring checks on a table level. These checks will be applied to tables. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultMonitoringMonthlyTableChecksPattern + """updateMonitoringMonthlyTableQualityPolicy New configuration of the default monthly monitoring checks on a table level. These checks will be applied to tables. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_partitioned_daily_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/update_partitioned_daily_table_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_table_check_patterns/update_default_partitioned_daily_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/update_partitioned_daily_table_quality_policy.py index 729da58b59..c41d3159c2 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_partitioned_daily_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/update_partitioned_daily_table_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/table/{patternName}/partitioned/daily".format( + "url": "api/policies/checks/table/{patternName}/partitioned/daily".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedDailyTableChecksPattern + """updatePartitionedDailyTableQualityPolicy New configuration of the default daily partitioned checks on a table level. These checks will be applied to tables. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedDailyTableChecksPattern + """updatePartitionedDailyTableQualityPolicy New configuration of the default daily partitioned checks on a table level. These checks will be applied to tables. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_partitioned_monthly_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/update_partitioned_monthly_table_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_table_check_patterns/update_default_partitioned_monthly_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/update_partitioned_monthly_table_quality_policy.py index 50d54093ef..cbedf591f3 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_partitioned_monthly_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/update_partitioned_monthly_table_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/table/{patternName}/partitioned/monthly".format( + "url": "api/policies/checks/table/{patternName}/partitioned/monthly".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedMonthlyTableChecksPattern + """updatePartitionedMonthlyTableQualityPolicy New configuration of the default monthly partitioned checks on a table level. These checks will be applied to tables. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultPartitionedMonthlyTableChecksPattern + """updatePartitionedMonthlyTableQualityPolicy New configuration of the default monthly partitioned checks on a table level. These checks will be applied to tables. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_profiling_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/update_profiling_table_quality_policy.py similarity index 94% rename from distribution/python/dqops/client/api/default_table_check_patterns/update_default_profiling_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/update_profiling_table_quality_policy.py index 2f23356d4d..07ea28ae2c 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_profiling_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/update_profiling_table_quality_policy.py @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/table/{patternName}/profiling".format( + "url": "api/policies/checks/table/{patternName}/profiling".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,7 +56,7 @@ def sync_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultProfilingTableChecksPattern + """updateProfilingTableQualityPolicy New configuration of the default profiling checks on a table level. These checks will be applied to tables. @@ -92,7 +92,7 @@ async def asyncio_detailed( client: AuthenticatedClient, json_body: CheckContainerModel, ) -> Response[Any]: - """updateDefaultProfilingTableChecksPattern + """updateProfilingTableQualityPolicy New configuration of the default profiling checks on a table level. These checks will be applied to tables. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_table_checks_pattern.py b/distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy.py similarity index 75% rename from distribution/python/dqops/client/api/default_table_check_patterns/update_default_table_checks_pattern.py rename to distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy.py index e78ac23288..31e0e9b73d 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_table_checks_pattern.py +++ b/distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy.py @@ -5,14 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_table_checks_pattern_model import DefaultTableChecksPatternModel +from ...models.table_quality_policy_model import TableQualityPolicyModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultTableChecksPatternModel, + json_body: TableQualityPolicyModel, ) -> Dict[str, Any]: pass @@ -21,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/table/{patternName}".format( + "url": "api/policies/checks/table/{patternName}".format( patternName=pattern_name, ), "json": json_json_body, @@ -54,15 +54,17 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternModel, + json_body: TableQualityPolicyModel, ) -> Response[Any]: - """updateDefaultTableChecksPattern + """updateTableQualityPolicy - Updates an default table-level checks pattern by saving a full specification object + Updates an default table-level checks pattern (data quality policy) by saving a full specification + object Args: pattern_name (str): - json_body (DefaultTableChecksPatternModel): Default table-level checks pattern model + json_body (TableQualityPolicyModel): Default table-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -88,15 +90,17 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternModel, + json_body: TableQualityPolicyModel, ) -> Response[Any]: - """updateDefaultTableChecksPattern + """updateTableQualityPolicy - Updates an default table-level checks pattern by saving a full specification object + Updates an default table-level checks pattern (data quality policy) by saving a full specification + object Args: pattern_name (str): - json_body (DefaultTableChecksPatternModel): Default table-level checks pattern model + json_body (TableQualityPolicyModel): Default table-level checks pattern (data quality + policy) model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_table_checks_pattern_target.py b/distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy_target.py similarity index 74% rename from distribution/python/dqops/client/api/default_table_check_patterns/update_default_table_checks_pattern_target.py rename to distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy_target.py index c646d5ca0c..f5e168e38d 100644 --- a/distribution/python/dqops/client/api/default_table_check_patterns/update_default_table_checks_pattern_target.py +++ b/distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy_target.py @@ -5,16 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.default_table_checks_pattern_list_model import ( - DefaultTableChecksPatternListModel, -) +from ...models.table_quality_policy_list_model import TableQualityPolicyListModel from ...types import Response def _get_kwargs( pattern_name: str, *, - json_body: DefaultTableChecksPatternListModel, + json_body: TableQualityPolicyListModel, ) -> Dict[str, Any]: pass @@ -23,7 +21,7 @@ def _get_kwargs( return { "method": "put", - "url": "api/default/checks/table/{patternName}/target".format( + "url": "api/policies/checks/table/{patternName}/target".format( patternName=pattern_name, ), "json": json_json_body, @@ -56,16 +54,16 @@ def sync_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternListModel, + json_body: TableQualityPolicyListModel, ) -> Response[Any]: - """updateDefaultTableChecksPatternTarget + """updateTableQualityPolicyTarget - Updates an default table-level checks pattern, changing only the target object + Updates an default table-level checks pattern (data quality policy), changing only the target object Args: pattern_name (str): - json_body (DefaultTableChecksPatternListModel): Default table-level checks pattern list - model + json_body (TableQualityPolicyListModel): Default table-level checks pattern (data quality + policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -91,16 +89,16 @@ async def asyncio_detailed( pattern_name: str, *, client: AuthenticatedClient, - json_body: DefaultTableChecksPatternListModel, + json_body: TableQualityPolicyListModel, ) -> Response[Any]: - """updateDefaultTableChecksPatternTarget + """updateTableQualityPolicyTarget - Updates an default table-level checks pattern, changing only the target object + Updates an default table-level checks pattern (data quality policy), changing only the target object Args: pattern_name (str): - json_body (DefaultTableChecksPatternListModel): Default table-level checks pattern list - model + json_body (TableQualityPolicyListModel): Default table-level checks pattern (data quality + policy) list model Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/users/create_user.py b/distribution/python/dqops/client/api/users/create_user.py index dc54e707d3..6f19e89834 100644 --- a/distribution/python/dqops/client/api/users/create_user.py +++ b/distribution/python/dqops/client/api/users/create_user.py @@ -5,13 +5,13 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.dqo_cloud_user_model import DqoCloudUserModel +from ...models.dqo_user_roles_model import DqoUserRolesModel from ...types import Response def _get_kwargs( *, - json_body: DqoCloudUserModel, + json_body: DqoUserRolesModel, ) -> Dict[str, Any]: pass @@ -50,15 +50,15 @@ def _build_response( def sync_detailed( *, client: AuthenticatedClient, - json_body: DqoCloudUserModel, + json_body: DqoUserRolesModel, ) -> Response[Any]: """createUser Creates (adds) a new user to a multi-user account. Args: - json_body (DqoCloudUserModel): DQOps Cloud user model - identifies a user in a multi-user - DQOps deployment. + json_body (DqoUserRolesModel): DQOps user model - identifies a user in a multi-user DQOps + deployment and the user's roles. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -82,15 +82,15 @@ def sync_detailed( async def asyncio_detailed( *, client: AuthenticatedClient, - json_body: DqoCloudUserModel, + json_body: DqoUserRolesModel, ) -> Response[Any]: """createUser Creates (adds) a new user to a multi-user account. Args: - json_body (DqoCloudUserModel): DQOps Cloud user model - identifies a user in a multi-user - DQOps deployment. + json_body (DqoUserRolesModel): DQOps user model - identifies a user in a multi-user DQOps + deployment and the user's roles. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/api/users/get_all_users.py b/distribution/python/dqops/client/api/users/get_all_users.py index 9c5b86f611..6929239047 100644 --- a/distribution/python/dqops/client/api/users/get_all_users.py +++ b/distribution/python/dqops/client/api/users/get_all_users.py @@ -5,7 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.dqo_cloud_user_model import DqoCloudUserModel +from ...models.dqo_user_roles_model import DqoUserRolesModel from ...types import Response @@ -21,12 +21,12 @@ def _get_kwargs() -> Dict[str, Any]: def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[List["DqoCloudUserModel"]]: +) -> Optional[List["DqoUserRolesModel"]]: if response.status_code == HTTPStatus.OK: response_200 = [] _response_200 = response.json() for response_200_item_data in _response_200: - response_200_item = DqoCloudUserModel.from_dict(response_200_item_data) + response_200_item = DqoUserRolesModel.from_dict(response_200_item_data) response_200.append(response_200_item) @@ -39,7 +39,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[List["DqoCloudUserModel"]]: +) -> Response[List["DqoUserRolesModel"]]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -51,7 +51,7 @@ def _build_response( def sync_detailed( *, client: AuthenticatedClient, -) -> Response[List["DqoCloudUserModel"]]: +) -> Response[List["DqoUserRolesModel"]]: """getAllUsers Returns a list of all users. @@ -61,7 +61,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[List['DqoCloudUserModel']] + Response[List['DqoUserRolesModel']] """ kwargs = _get_kwargs() @@ -76,7 +76,7 @@ def sync_detailed( def sync( *, client: AuthenticatedClient, -) -> Optional[List["DqoCloudUserModel"]]: +) -> Optional[List["DqoUserRolesModel"]]: """getAllUsers Returns a list of all users. @@ -86,7 +86,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - List['DqoCloudUserModel'] + List['DqoUserRolesModel'] """ return sync_detailed( @@ -97,7 +97,7 @@ def sync( async def asyncio_detailed( *, client: AuthenticatedClient, -) -> Response[List["DqoCloudUserModel"]]: +) -> Response[List["DqoUserRolesModel"]]: """getAllUsers Returns a list of all users. @@ -107,7 +107,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[List['DqoCloudUserModel']] + Response[List['DqoUserRolesModel']] """ kwargs = _get_kwargs() @@ -120,7 +120,7 @@ async def asyncio_detailed( async def asyncio( *, client: AuthenticatedClient, -) -> Optional[List["DqoCloudUserModel"]]: +) -> Optional[List["DqoUserRolesModel"]]: """getAllUsers Returns a list of all users. @@ -130,7 +130,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - List['DqoCloudUserModel'] + List['DqoUserRolesModel'] """ return ( diff --git a/distribution/python/dqops/client/api/users/get_user.py b/distribution/python/dqops/client/api/users/get_user.py index 455323f81b..65b7ca5e4f 100644 --- a/distribution/python/dqops/client/api/users/get_user.py +++ b/distribution/python/dqops/client/api/users/get_user.py @@ -5,7 +5,7 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.dqo_cloud_user_model import DqoCloudUserModel +from ...models.dqo_user_roles_model import DqoUserRolesModel from ...types import Response @@ -25,9 +25,9 @@ def _get_kwargs( def _parse_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Optional[DqoCloudUserModel]: +) -> Optional[DqoUserRolesModel]: if response.status_code == HTTPStatus.OK: - response_200 = DqoCloudUserModel.from_dict(response.json()) + response_200 = DqoUserRolesModel.from_dict(response.json()) return response_200 if client.raise_on_unexpected_status: @@ -38,7 +38,7 @@ def _parse_response( def _build_response( *, client: Union[AuthenticatedClient, Client], response: httpx.Response -) -> Response[DqoCloudUserModel]: +) -> Response[DqoUserRolesModel]: return Response( status_code=HTTPStatus(response.status_code), content=response.content, @@ -51,7 +51,7 @@ def sync_detailed( email: str, *, client: AuthenticatedClient, -) -> Response[DqoCloudUserModel]: +) -> Response[DqoUserRolesModel]: """getUser Returns the user model that describes the role of a user identified by an email @@ -64,7 +64,7 @@ def sync_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DqoCloudUserModel] + Response[DqoUserRolesModel] """ kwargs = _get_kwargs( @@ -82,7 +82,7 @@ def sync( email: str, *, client: AuthenticatedClient, -) -> Optional[DqoCloudUserModel]: +) -> Optional[DqoUserRolesModel]: """getUser Returns the user model that describes the role of a user identified by an email @@ -95,7 +95,7 @@ def sync( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DqoCloudUserModel + DqoUserRolesModel """ return sync_detailed( @@ -108,7 +108,7 @@ async def asyncio_detailed( email: str, *, client: AuthenticatedClient, -) -> Response[DqoCloudUserModel]: +) -> Response[DqoUserRolesModel]: """getUser Returns the user model that describes the role of a user identified by an email @@ -121,7 +121,7 @@ async def asyncio_detailed( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - Response[DqoCloudUserModel] + Response[DqoUserRolesModel] """ kwargs = _get_kwargs( @@ -137,7 +137,7 @@ async def asyncio( email: str, *, client: AuthenticatedClient, -) -> Optional[DqoCloudUserModel]: +) -> Optional[DqoUserRolesModel]: """getUser Returns the user model that describes the role of a user identified by an email @@ -150,7 +150,7 @@ async def asyncio( httpx.TimeoutException: If the request takes longer than Client.timeout. Returns: - DqoCloudUserModel + DqoUserRolesModel """ return ( diff --git a/distribution/python/dqops/client/api/users/update_user.py b/distribution/python/dqops/client/api/users/update_user.py index e2470d711b..6060b16dc9 100644 --- a/distribution/python/dqops/client/api/users/update_user.py +++ b/distribution/python/dqops/client/api/users/update_user.py @@ -5,14 +5,14 @@ from ... import errors from ...client import AuthenticatedClient, Client -from ...models.dqo_cloud_user_model import DqoCloudUserModel +from ...models.dqo_user_roles_model import DqoUserRolesModel from ...types import Response def _get_kwargs( email: str, *, - json_body: DqoCloudUserModel, + json_body: DqoUserRolesModel, ) -> Dict[str, Any]: pass @@ -54,7 +54,7 @@ def sync_detailed( email: str, *, client: AuthenticatedClient, - json_body: DqoCloudUserModel, + json_body: DqoUserRolesModel, ) -> Response[Any]: """updateUser @@ -62,8 +62,8 @@ def sync_detailed( Args: email (str): - json_body (DqoCloudUserModel): DQOps Cloud user model - identifies a user in a multi-user - DQOps deployment. + json_body (DqoUserRolesModel): DQOps user model - identifies a user in a multi-user DQOps + deployment and the user's roles. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. @@ -89,7 +89,7 @@ async def asyncio_detailed( email: str, *, client: AuthenticatedClient, - json_body: DqoCloudUserModel, + json_body: DqoUserRolesModel, ) -> Response[Any]: """updateUser @@ -97,8 +97,8 @@ async def asyncio_detailed( Args: email (str): - json_body (DqoCloudUserModel): DQOps Cloud user model - identifies a user in a multi-user - DQOps deployment. + json_body (DqoUserRolesModel): DQOps user model - identifies a user in a multi-user DQOps + deployment and the user's roles. Raises: errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. diff --git a/distribution/python/dqops/client/models/__init__.py b/distribution/python/dqops/client/models/__init__.py index f30e749345..75b70d6d1f 100644 --- a/distribution/python/dqops/client/models/__init__.py +++ b/distribution/python/dqops/client/models/__init__.py @@ -438,7 +438,6 @@ from .column_datetime_profiling_checks_spec_custom_checks import ( ColumnDatetimeProfilingChecksSpecCustomChecks, ) -from .column_default_checks_pattern_spec import ColumnDefaultChecksPatternSpec from .column_detected_datatype_in_text_check_spec import ( ColumnDetectedDatatypeInTextCheckSpec, ) @@ -567,7 +566,10 @@ from .column_invalid_uuid_format_percent_check_spec import ( ColumnInvalidUuidFormatPercentCheckSpec, ) +from .column_lineage_source_spec import ColumnLineageSourceSpec +from .column_lineage_source_spec_properties import ColumnLineageSourceSpecProperties from .column_list_model import ColumnListModel +from .column_list_model_advanced_properties import ColumnListModelAdvancedProperties from .column_max_anomaly_differencing_check_spec import ( ColumnMaxAnomalyDifferencingCheckSpec, ) @@ -974,6 +976,9 @@ from .column_profiling_check_categories_spec_custom import ( ColumnProfilingCheckCategoriesSpecCustom, ) +from .column_quality_policy_list_model import ColumnQualityPolicyListModel +from .column_quality_policy_model import ColumnQualityPolicyModel +from .column_quality_policy_spec import ColumnQualityPolicySpec from .column_range_max_value_sensor_parameters_spec import ( ColumnRangeMaxValueSensorParametersSpec, ) @@ -1028,6 +1033,7 @@ ) from .column_schema_type_changed_check_spec import ColumnSchemaTypeChangedCheckSpec from .column_spec import ColumnSpec +from .column_spec_advanced_properties import ColumnSpecAdvancedProperties from .column_sql_aggregate_expression_check_spec import ( ColumnSqlAggregateExpressionCheckSpec, ) @@ -1374,7 +1380,9 @@ from .compression_type import CompressionType from .connection_incident_grouping_spec import ConnectionIncidentGroupingSpec from .connection_model import ConnectionModel +from .connection_model_advanced_properties import ConnectionModelAdvancedProperties from .connection_spec import ConnectionSpec +from .connection_spec_advanced_properties import ConnectionSpecAdvancedProperties from .connection_specification_model import ConnectionSpecificationModel from .connection_test_model import ConnectionTestModel from .connection_test_status import ConnectionTestStatus @@ -1402,14 +1410,8 @@ from .databricks_parameters_spec import DatabricksParametersSpec from .databricks_parameters_spec_properties import DatabricksParametersSpecProperties from .datetime_built_in_date_formats import DatetimeBuiltInDateFormats -from .default_column_checks_pattern_list_model import ( - DefaultColumnChecksPatternListModel, -) -from .default_column_checks_pattern_model import DefaultColumnChecksPatternModel from .default_rule_severity_level import DefaultRuleSeverityLevel from .default_schedules_spec import DefaultSchedulesSpec -from .default_table_checks_pattern_list_model import DefaultTableChecksPatternListModel -from .default_table_checks_pattern_model import DefaultTableChecksPatternModel from .delete_stored_data_queue_job_parameters import DeleteStoredDataQueueJobParameters from .delete_stored_data_queue_job_result import DeleteStoredDataQueueJobResult from .delete_stored_data_result import DeleteStoredDataResult @@ -1425,7 +1427,6 @@ DimensionCurrentDataQualityStatusModel, ) from .display_hint import DisplayHint -from .dqo_cloud_user_model import DqoCloudUserModel from .dqo_job_change_model import DqoJobChangeModel from .dqo_job_entry_parameters_model import DqoJobEntryParametersModel from .dqo_job_history_entry_model import DqoJobHistoryEntryModel @@ -1444,6 +1445,8 @@ ) from .dqo_user_profile_model import DqoUserProfileModel from .dqo_user_role import DqoUserRole +from .dqo_user_roles_model import DqoUserRolesModel +from .dqo_user_roles_model_data_domain_roles import DqoUserRolesModelDataDomainRoles from .duckdb_files_format_type import DuckdbFilesFormatType from .duckdb_parameters_spec import DuckdbParametersSpec from .duckdb_parameters_spec_directories import DuckdbParametersSpecDirectories @@ -1503,6 +1506,7 @@ from .json_format_type import JsonFormatType from .json_records_type import JsonRecordsType from .label_model import LabelModel +from .local_data_domain_model import LocalDataDomainModel from .max_count_rule_0_error_parameters_spec import MaxCountRule0ErrorParametersSpec from .max_count_rule_0_warning_parameters_spec import MaxCountRule0WarningParametersSpec from .max_count_rule_100_parameters_spec import MaxCountRule100ParametersSpec @@ -1807,9 +1811,26 @@ from .table_data_freshness_check_spec import TableDataFreshnessCheckSpec from .table_data_ingestion_delay_check_spec import TableDataIngestionDelayCheckSpec from .table_data_staleness_check_spec import TableDataStalenessCheckSpec -from .table_default_checks_pattern_spec import TableDefaultChecksPatternSpec +from .table_duplicate_record_count_check_spec import TableDuplicateRecordCountCheckSpec +from .table_duplicate_record_count_sensor_parameters_spec import ( + TableDuplicateRecordCountSensorParametersSpec, +) +from .table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, +) +from .table_duplicate_record_percent_sensor_parameters_spec import ( + TableDuplicateRecordPercentSensorParametersSpec, +) from .table_incident_grouping_spec import TableIncidentGroupingSpec +from .table_lineage_source_list_model import TableLineageSourceListModel +from .table_lineage_source_list_model_properties import ( + TableLineageSourceListModelProperties, +) +from .table_lineage_source_spec import TableLineageSourceSpec +from .table_lineage_source_spec_columns import TableLineageSourceSpecColumns +from .table_lineage_source_spec_properties import TableLineageSourceSpecProperties from .table_list_model import TableListModel +from .table_list_model_advanced_properties import TableListModelAdvancedProperties from .table_model import TableModel from .table_monitoring_check_categories_spec import TableMonitoringCheckCategoriesSpec from .table_monthly_monitoring_check_categories_spec import ( @@ -1842,6 +1863,9 @@ TableProfilingCheckCategoriesSpecCustom, ) from .table_profiling_setup_status_model import TableProfilingSetupStatusModel +from .table_quality_policy_list_model import TableQualityPolicyListModel +from .table_quality_policy_model import TableQualityPolicyModel +from .table_quality_policy_spec import TableQualityPolicySpec from .table_row_count_anomaly_differencing_check_spec import ( TableRowCountAnomalyDifferencingCheckSpec, ) @@ -1889,6 +1913,7 @@ ) from .table_schema_statistics_collectors_spec import TableSchemaStatisticsCollectorsSpec from .table_spec import TableSpec +from .table_spec_advanced_properties import TableSpecAdvancedProperties from .table_spec_columns import TableSpecColumns from .table_spec_groupings import TableSpecGroupings from .table_spec_table_comparisons import TableSpecTableComparisons @@ -1958,6 +1983,34 @@ from .table_timeliness_profiling_checks_spec_custom_checks import ( TableTimelinessProfilingChecksSpecCustomChecks, ) +from .table_uniqueness_daily_monitoring_checks_spec import ( + TableUniquenessDailyMonitoringChecksSpec, +) +from .table_uniqueness_daily_monitoring_checks_spec_custom_checks import ( + TableUniquenessDailyMonitoringChecksSpecCustomChecks, +) +from .table_uniqueness_daily_partition_checks_spec import ( + TableUniquenessDailyPartitionChecksSpec, +) +from .table_uniqueness_daily_partition_checks_spec_custom_checks import ( + TableUniquenessDailyPartitionChecksSpecCustomChecks, +) +from .table_uniqueness_monthly_monitoring_checks_spec import ( + TableUniquenessMonthlyMonitoringChecksSpec, +) +from .table_uniqueness_monthly_monitoring_checks_spec_custom_checks import ( + TableUniquenessMonthlyMonitoringChecksSpecCustomChecks, +) +from .table_uniqueness_monthly_partition_checks_spec import ( + TableUniquenessMonthlyPartitionChecksSpec, +) +from .table_uniqueness_monthly_partition_checks_spec_custom_checks import ( + TableUniquenessMonthlyPartitionChecksSpecCustomChecks, +) +from .table_uniqueness_profiling_checks_spec import TableUniquenessProfilingChecksSpec +from .table_uniqueness_profiling_checks_spec_custom_checks import ( + TableUniquenessProfilingChecksSpecCustomChecks, +) from .table_volume_daily_monitoring_checks_spec import ( TableVolumeDailyMonitoringChecksSpec, ) @@ -2194,7 +2247,6 @@ "ColumnDatetimeProfilingChecksSpec", "ColumnDatetimeProfilingChecksSpecCustomChecks", "ColumnDateValuesInFuturePercentCheckSpec", - "ColumnDefaultChecksPatternSpec", "ColumnDetectedDatatypeInTextCheckSpec", "ColumnDistinctCountAnomalyDifferencingCheckSpec", "ColumnDistinctCountAnomalyStationaryPartitionCheckSpec", @@ -2243,7 +2295,10 @@ "ColumnInvalidUsaZipcodePercentCheckSpec", "ColumnInvalidUuidFormatFoundCheckSpec", "ColumnInvalidUuidFormatPercentCheckSpec", + "ColumnLineageSourceSpec", + "ColumnLineageSourceSpecProperties", "ColumnListModel", + "ColumnListModelAdvancedProperties", "ColumnMaxAnomalyDifferencingCheckSpec", "ColumnMaxAnomalyStationaryCheckSpec", "ColumnMaxInRangeCheckSpec", @@ -2404,6 +2459,9 @@ "ColumnProfilingCheckCategoriesSpec", "ColumnProfilingCheckCategoriesSpecComparisons", "ColumnProfilingCheckCategoriesSpecCustom", + "ColumnQualityPolicyListModel", + "ColumnQualityPolicyModel", + "ColumnQualityPolicySpec", "ColumnRangeMaxValueSensorParametersSpec", "ColumnRangeMaxValueStatisticsCollectorSpec", "ColumnRangeMeanValueStatisticsCollectorSpec", @@ -2426,6 +2484,7 @@ "ColumnSchemaProfilingChecksSpecCustomChecks", "ColumnSchemaTypeChangedCheckSpec", "ColumnSpec", + "ColumnSpecAdvancedProperties", "ColumnSqlAggregatedExpressionSensorParametersSpec", "ColumnSqlAggregateExpressionCheckSpec", "ColumnSqlConditionFailedCheckSpec", @@ -2558,7 +2617,9 @@ "CompressionType", "ConnectionIncidentGroupingSpec", "ConnectionModel", + "ConnectionModelAdvancedProperties", "ConnectionSpec", + "ConnectionSpecAdvancedProperties", "ConnectionSpecificationModel", "ConnectionTestModel", "ConnectionTestStatus", @@ -2584,12 +2645,8 @@ "DataGroupingDimensionSpec", "DataTypeCategory", "DatetimeBuiltInDateFormats", - "DefaultColumnChecksPatternListModel", - "DefaultColumnChecksPatternModel", "DefaultRuleSeverityLevel", "DefaultSchedulesSpec", - "DefaultTableChecksPatternListModel", - "DefaultTableChecksPatternModel", "DeleteStoredDataQueueJobParameters", "DeleteStoredDataQueueJobResult", "DeleteStoredDataResult", @@ -2599,7 +2656,6 @@ "DetectedDatatypeEqualsRuleParametersSpec", "DimensionCurrentDataQualityStatusModel", "DisplayHint", - "DqoCloudUserModel", "DqoJobChangeModel", "DqoJobEntryParametersModel", "DqoJobHistoryEntryModel", @@ -2614,6 +2670,8 @@ "DqoSettingsModelPropertiesAdditionalProperty", "DqoUserProfileModel", "DqoUserRole", + "DqoUserRolesModel", + "DqoUserRolesModelDataDomainRoles", "DuckdbFilesFormatType", "DuckdbParametersSpec", "DuckdbParametersSpecDirectories", @@ -2669,6 +2727,7 @@ "JsonFormatType", "JsonRecordsType", "LabelModel", + "LocalDataDomainModel", "MaxCountRule0ErrorParametersSpec", "MaxCountRule0WarningParametersSpec", "MaxCountRule100ParametersSpec", @@ -2849,9 +2908,18 @@ "TableDataFreshnessCheckSpec", "TableDataIngestionDelayCheckSpec", "TableDataStalenessCheckSpec", - "TableDefaultChecksPatternSpec", + "TableDuplicateRecordCountCheckSpec", + "TableDuplicateRecordCountSensorParametersSpec", + "TableDuplicateRecordPercentCheckSpec", + "TableDuplicateRecordPercentSensorParametersSpec", "TableIncidentGroupingSpec", + "TableLineageSourceListModel", + "TableLineageSourceListModelProperties", + "TableLineageSourceSpec", + "TableLineageSourceSpecColumns", + "TableLineageSourceSpecProperties", "TableListModel", + "TableListModelAdvancedProperties", "TableModel", "TableMonitoringCheckCategoriesSpec", "TableMonthlyMonitoringCheckCategoriesSpec", @@ -2868,6 +2936,9 @@ "TableProfilingCheckCategoriesSpecComparisons", "TableProfilingCheckCategoriesSpecCustom", "TableProfilingSetupStatusModel", + "TableQualityPolicyListModel", + "TableQualityPolicyModel", + "TableQualityPolicySpec", "TableRowCountAnomalyDifferencingCheckSpec", "TableRowCountAnomalyStationaryPartitionCheckSpec", "TableRowCountChange1DayCheckSpec", @@ -2889,6 +2960,7 @@ "TableSchemaProfilingChecksSpecCustomChecks", "TableSchemaStatisticsCollectorsSpec", "TableSpec", + "TableSpecAdvancedProperties", "TableSpecColumns", "TableSpecGroupings", "TableSpecTableComparisons", @@ -2916,6 +2988,16 @@ "TableTimelinessPartitionReloadLagSensorParametersSpec", "TableTimelinessProfilingChecksSpec", "TableTimelinessProfilingChecksSpecCustomChecks", + "TableUniquenessDailyMonitoringChecksSpec", + "TableUniquenessDailyMonitoringChecksSpecCustomChecks", + "TableUniquenessDailyPartitionChecksSpec", + "TableUniquenessDailyPartitionChecksSpecCustomChecks", + "TableUniquenessMonthlyMonitoringChecksSpec", + "TableUniquenessMonthlyMonitoringChecksSpecCustomChecks", + "TableUniquenessMonthlyPartitionChecksSpec", + "TableUniquenessMonthlyPartitionChecksSpecCustomChecks", + "TableUniquenessProfilingChecksSpec", + "TableUniquenessProfilingChecksSpecCustomChecks", "TableVolumeDailyMonitoringChecksSpec", "TableVolumeDailyMonitoringChecksSpecCustomChecks", "TableVolumeDailyPartitionedChecksSpec", diff --git a/distribution/python/dqops/client/models/check_mining_parameters_model.py b/distribution/python/dqops/client/models/check_mining_parameters_model.py index 97c73852c3..340dd0cc3a 100644 --- a/distribution/python/dqops/client/models/check_mining_parameters_model.py +++ b/distribution/python/dqops/client/models/check_mining_parameters_model.py @@ -25,8 +25,10 @@ class CheckMiningParametersModel: copy_disabled_profiling_checks (Union[Unset, bool]): Copy also the configuration of profiling checks that are disabled. copy_profiling_checks (Union[Unset, bool]): Copy the configuration of valid profiling checks. - propose_default_checks (Union[Unset, bool]): Propose the rules for default checks that were activated using data - quality check patterns (policies). The default value of this parameter is 'true'. + reconfigure_policy_enabled_checks (Union[Unset, bool]): Propose the rules for default checks that were activated + using data quality check patterns (policies). The default value of this parameter is 'true'. + propose_checks_from_statistics (Union[Unset, bool]): Propose the configuration of data quality checks from + statistics. propose_minimum_row_count (Union[Unset, bool]): Propose the default configuration of the minimum row count for monitoring checks (full table scans). The default value of this parameter is 'true'. propose_column_count (Union[Unset, bool]): Propose the default configuration of the column count check. The @@ -85,9 +87,9 @@ class CheckMiningParametersModel: checks. Whitespace checks detect common data quality issues with storing text representations of null values, such as 'null', 'None', 'n/a' and other texts that should be stored as regular NULL values. The default value of this parameter is 'true'. - apply_pii_checks (Union[Unset, bool]): Apply the rules to the Personal Identifiable Information checks - (sensitive data), but only when the checks were run as profiling checks activated manually, or by activating a - data quality check pattern that scans all columns for PII data. The default value of this parameter is 'true'. + apply_pii_checks (Union[Unset, bool]): Applies rules to Personal Identifiable Information checks (sensitive + data), but only when the checks were activated manually as profiling checks, or through a data quality check + pattern that scans all columns for PII data. The default value of this parameter is 'true'. propose_custom_checks (Union[Unset, bool]): Propose the default configuration for custom checks that use built- in data quality rules. The default value of this parameter is 'true'. fail_checks_at_percent_error_rows (Union[Unset, float]): The percentage value captured by a profiling check (for @@ -104,7 +106,8 @@ class CheckMiningParametersModel: copy_failed_profiling_checks: Union[Unset, bool] = UNSET copy_disabled_profiling_checks: Union[Unset, bool] = UNSET copy_profiling_checks: Union[Unset, bool] = UNSET - propose_default_checks: Union[Unset, bool] = UNSET + reconfigure_policy_enabled_checks: Union[Unset, bool] = UNSET + propose_checks_from_statistics: Union[Unset, bool] = UNSET propose_minimum_row_count: Union[Unset, bool] = UNSET propose_column_count: Union[Unset, bool] = UNSET propose_timeliness_checks: Union[Unset, bool] = UNSET @@ -143,7 +146,8 @@ def to_dict(self) -> Dict[str, Any]: copy_failed_profiling_checks = self.copy_failed_profiling_checks copy_disabled_profiling_checks = self.copy_disabled_profiling_checks copy_profiling_checks = self.copy_profiling_checks - propose_default_checks = self.propose_default_checks + reconfigure_policy_enabled_checks = self.reconfigure_policy_enabled_checks + propose_checks_from_statistics = self.propose_checks_from_statistics propose_minimum_row_count = self.propose_minimum_row_count propose_column_count = self.propose_column_count propose_timeliness_checks = self.propose_timeliness_checks @@ -193,8 +197,14 @@ def to_dict(self) -> Dict[str, Any]: ) if copy_profiling_checks is not UNSET: field_dict["copy_profiling_checks"] = copy_profiling_checks - if propose_default_checks is not UNSET: - field_dict["propose_default_checks"] = propose_default_checks + if reconfigure_policy_enabled_checks is not UNSET: + field_dict["reconfigure_policy_enabled_checks"] = ( + reconfigure_policy_enabled_checks + ) + if propose_checks_from_statistics is not UNSET: + field_dict["propose_checks_from_statistics"] = ( + propose_checks_from_statistics + ) if propose_minimum_row_count is not UNSET: field_dict["propose_minimum_row_count"] = propose_minimum_row_count if propose_column_count is not UNSET: @@ -280,7 +290,11 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: copy_profiling_checks = d.pop("copy_profiling_checks", UNSET) - propose_default_checks = d.pop("propose_default_checks", UNSET) + reconfigure_policy_enabled_checks = d.pop( + "reconfigure_policy_enabled_checks", UNSET + ) + + propose_checks_from_statistics = d.pop("propose_checks_from_statistics", UNSET) propose_minimum_row_count = d.pop("propose_minimum_row_count", UNSET) @@ -348,7 +362,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: copy_failed_profiling_checks=copy_failed_profiling_checks, copy_disabled_profiling_checks=copy_disabled_profiling_checks, copy_profiling_checks=copy_profiling_checks, - propose_default_checks=propose_default_checks, + reconfigure_policy_enabled_checks=reconfigure_policy_enabled_checks, + propose_checks_from_statistics=propose_checks_from_statistics, propose_minimum_row_count=propose_minimum_row_count, propose_column_count=propose_column_count, propose_timeliness_checks=propose_timeliness_checks, diff --git a/distribution/python/dqops/client/models/collect_statistics_on_table_queue_job_parameters.py b/distribution/python/dqops/client/models/collect_statistics_on_table_queue_job_parameters.py index eb22c0a969..2b4ecb0f32 100644 --- a/distribution/python/dqops/client/models/collect_statistics_on_table_queue_job_parameters.py +++ b/distribution/python/dqops/client/models/collect_statistics_on_table_queue_job_parameters.py @@ -27,6 +27,9 @@ class CollectStatisticsOnTableQueueJobParameters: table (Union[Unset, PhysicalTableName]): statistics_collector_search_filters (Union[Unset, StatisticsCollectorSearchFilters]): data_scope (Union[Unset, StatisticsDataScope]): + samples_limit (Union[Unset, int]): The default limit of column samples that are collected. + configure_table (Union[Unset, bool]): Turns on a special mode of collecting statistics that will configure the + timestamp and ID columns. It should be used only during the first statistics collection. dummy_sensor_execution (Union[Unset, bool]): Boolean flag that enables a dummy statistics collection (sensors are executed, but the statistics results are not written to the parquet files). collect_statistics_result (Union[Unset, CollectStatisticsResult]): Returns the result with the summary of the @@ -40,6 +43,8 @@ class CollectStatisticsOnTableQueueJobParameters: Unset, "StatisticsCollectorSearchFilters" ] = UNSET data_scope: Union[Unset, StatisticsDataScope] = UNSET + samples_limit: Union[Unset, int] = UNSET + configure_table: Union[Unset, bool] = UNSET dummy_sensor_execution: Union[Unset, bool] = UNSET collect_statistics_result: Union[Unset, "CollectStatisticsResult"] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) @@ -61,6 +66,8 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.data_scope, Unset): data_scope = self.data_scope.value + samples_limit = self.samples_limit + configure_table = self.configure_table dummy_sensor_execution = self.dummy_sensor_execution collect_statistics_result: Union[Unset, Dict[str, Any]] = UNSET if not isinstance(self.collect_statistics_result, Unset): @@ -81,6 +88,10 @@ def to_dict(self) -> Dict[str, Any]: ) if data_scope is not UNSET: field_dict["data_scope"] = data_scope + if samples_limit is not UNSET: + field_dict["samples_limit"] = samples_limit + if configure_table is not UNSET: + field_dict["configure_table"] = configure_table if dummy_sensor_execution is not UNSET: field_dict["dummy_sensor_execution"] = dummy_sensor_execution if collect_statistics_result is not UNSET: @@ -130,6 +141,10 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: data_scope = StatisticsDataScope(_data_scope) + samples_limit = d.pop("samples_limit", UNSET) + + configure_table = d.pop("configure_table", UNSET) + dummy_sensor_execution = d.pop("dummy_sensor_execution", UNSET) _collect_statistics_result = d.pop("collect_statistics_result", UNSET) @@ -147,6 +162,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: table=table, statistics_collector_search_filters=statistics_collector_search_filters, data_scope=data_scope, + samples_limit=samples_limit, + configure_table=configure_table, dummy_sensor_execution=dummy_sensor_execution, collect_statistics_result=collect_statistics_result, ) diff --git a/distribution/python/dqops/client/models/collect_statistics_queue_job_parameters.py b/distribution/python/dqops/client/models/collect_statistics_queue_job_parameters.py index 83528e1d21..8396ec02b6 100644 --- a/distribution/python/dqops/client/models/collect_statistics_queue_job_parameters.py +++ b/distribution/python/dqops/client/models/collect_statistics_queue_job_parameters.py @@ -22,6 +22,9 @@ class CollectStatisticsQueueJobParameters: Attributes: statistics_collector_search_filters (Union[Unset, StatisticsCollectorSearchFilters]): data_scope (Union[Unset, StatisticsDataScope]): + configure_table (Union[Unset, bool]): Turns on a special mode of collecting statistics that will configure the + timestamp and ID columns. It should be used only during the first statistics collection. + samples_limit (Union[Unset, int]): The default limit of column samples that are collected. dummy_sensor_execution (Union[Unset, bool]): Boolean flag that enables a dummy statistics collection (sensors are executed, but the statistics results are not written to the parquet files). collect_statistics_result (Union[Unset, CollectStatisticsResult]): Returns the result with the summary of the @@ -32,6 +35,8 @@ class CollectStatisticsQueueJobParameters: Unset, "StatisticsCollectorSearchFilters" ] = UNSET data_scope: Union[Unset, StatisticsDataScope] = UNSET + configure_table: Union[Unset, bool] = UNSET + samples_limit: Union[Unset, int] = UNSET dummy_sensor_execution: Union[Unset, bool] = UNSET collect_statistics_result: Union[Unset, "CollectStatisticsResult"] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) @@ -47,6 +52,8 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.data_scope, Unset): data_scope = self.data_scope.value + configure_table = self.configure_table + samples_limit = self.samples_limit dummy_sensor_execution = self.dummy_sensor_execution collect_statistics_result: Union[Unset, Dict[str, Any]] = UNSET if not isinstance(self.collect_statistics_result, Unset): @@ -61,6 +68,10 @@ def to_dict(self) -> Dict[str, Any]: ) if data_scope is not UNSET: field_dict["data_scope"] = data_scope + if configure_table is not UNSET: + field_dict["configure_table"] = configure_table + if samples_limit is not UNSET: + field_dict["samples_limit"] = samples_limit if dummy_sensor_execution is not UNSET: field_dict["dummy_sensor_execution"] = dummy_sensor_execution if collect_statistics_result is not UNSET: @@ -98,6 +109,10 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: data_scope = StatisticsDataScope(_data_scope) + configure_table = d.pop("configure_table", UNSET) + + samples_limit = d.pop("samples_limit", UNSET) + dummy_sensor_execution = d.pop("dummy_sensor_execution", UNSET) _collect_statistics_result = d.pop("collect_statistics_result", UNSET) @@ -112,6 +127,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: collect_statistics_queue_job_parameters = cls( statistics_collector_search_filters=statistics_collector_search_filters, data_scope=data_scope, + configure_table=configure_table, + samples_limit=samples_limit, dummy_sensor_execution=dummy_sensor_execution, collect_statistics_result=collect_statistics_result, ) diff --git a/distribution/python/dqops/client/models/column_lineage_source_spec.py b/distribution/python/dqops/client/models/column_lineage_source_spec.py new file mode 100644 index 0000000000..f1ec12f275 --- /dev/null +++ b/distribution/python/dqops/client/models/column_lineage_source_spec.py @@ -0,0 +1,89 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.column_lineage_source_spec_properties import ( + ColumnLineageSourceSpecProperties, + ) + + +T = TypeVar("T", bound="ColumnLineageSourceSpec") + + +@_attrs_define +class ColumnLineageSourceSpec: + """ + Attributes: + source_columns (Union[Unset, List[str]]): A list of source columns from the source table name from which this + column receives data. + properties (Union[Unset, ColumnLineageSourceSpecProperties]): A dictionary of mapping properties stored as a + key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external + data lineage sources can use it to store mapping information. + """ + + source_columns: Union[Unset, List[str]] = UNSET + properties: Union[Unset, "ColumnLineageSourceSpecProperties"] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + source_columns: Union[Unset, List[str]] = UNSET + if not isinstance(self.source_columns, Unset): + source_columns = self.source_columns + + properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.properties, Unset): + properties = self.properties.to_dict() + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if source_columns is not UNSET: + field_dict["source_columns"] = source_columns + if properties is not UNSET: + field_dict["properties"] = properties + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.column_lineage_source_spec_properties import ( + ColumnLineageSourceSpecProperties, + ) + + d = src_dict.copy() + source_columns = cast(List[str], d.pop("source_columns", UNSET)) + + _properties = d.pop("properties", UNSET) + properties: Union[Unset, ColumnLineageSourceSpecProperties] + if isinstance(_properties, Unset): + properties = UNSET + else: + properties = ColumnLineageSourceSpecProperties.from_dict(_properties) + + column_lineage_source_spec = cls( + source_columns=source_columns, + properties=properties, + ) + + column_lineage_source_spec.additional_properties = d + return column_lineage_source_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/column_lineage_source_spec_properties.py b/distribution/python/dqops/client/models/column_lineage_source_spec_properties.py new file mode 100644 index 0000000000..3abf1f2bea --- /dev/null +++ b/distribution/python/dqops/client/models/column_lineage_source_spec_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="ColumnLineageSourceSpecProperties") + + +@_attrs_define +class ColumnLineageSourceSpecProperties: + """A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are + importing data lineage mappings from external data lineage sources can use it to store mapping information. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + column_lineage_source_spec_properties = cls() + + column_lineage_source_spec_properties.additional_properties = d + return column_lineage_source_spec_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/column_list_model.py b/distribution/python/dqops/client/models/column_list_model.py index 1179adc7eb..5e3785b83e 100644 --- a/distribution/python/dqops/client/models/column_list_model.py +++ b/distribution/python/dqops/client/models/column_list_model.py @@ -10,6 +10,9 @@ from ..models.column_current_data_quality_status_model import ( ColumnCurrentDataQualityStatusModel, ) + from ..models.column_list_model_advanced_properties import ( + ColumnListModelAdvancedProperties, + ) from ..models.column_type_snapshot_spec import ColumnTypeSnapshotSpec from ..models.delete_stored_data_queue_job_parameters import ( DeleteStoredDataQueueJobParameters, @@ -61,6 +64,8 @@ class ColumnListModel: identifies which checks on which tables and columns should be executed. collect_statistics_job_template (Union[Unset, StatisticsCollectorSearchFilters]): data_clean_job_template (Union[Unset, DeleteStoredDataQueueJobParameters]): + advanced_properties (Union[Unset, ColumnListModelAdvancedProperties]): A dictionary of advanced properties that + can be used for e.g. to support mapping data to data catalogs, a key/value dictionary. can_edit (Union[Unset, bool]): Boolean flag that decides if the current user can update or delete the column. can_collect_statistics (Union[Unset, bool]): Boolean flag that decides if the current user can collect statistics. @@ -90,6 +95,7 @@ class ColumnListModel: Unset, "StatisticsCollectorSearchFilters" ] = UNSET data_clean_job_template: Union[Unset, "DeleteStoredDataQueueJobParameters"] = UNSET + advanced_properties: Union[Unset, "ColumnListModelAdvancedProperties"] = UNSET can_edit: Union[Unset, bool] = UNSET can_collect_statistics: Union[Unset, bool] = UNSET can_run_checks: Union[Unset, bool] = UNSET @@ -155,6 +161,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.data_clean_job_template, Unset): data_clean_job_template = self.data_clean_job_template.to_dict() + advanced_properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.advanced_properties, Unset): + advanced_properties = self.advanced_properties.to_dict() + can_edit = self.can_edit can_collect_statistics = self.can_collect_statistics can_run_checks = self.can_run_checks @@ -217,6 +227,8 @@ def to_dict(self) -> Dict[str, Any]: ) if data_clean_job_template is not UNSET: field_dict["data_clean_job_template"] = data_clean_job_template + if advanced_properties is not UNSET: + field_dict["advanced_properties"] = advanced_properties if can_edit is not UNSET: field_dict["can_edit"] = can_edit if can_collect_statistics is not UNSET: @@ -234,6 +246,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.column_current_data_quality_status_model import ( ColumnCurrentDataQualityStatusModel, ) + from ..models.column_list_model_advanced_properties import ( + ColumnListModelAdvancedProperties, + ) from ..models.column_type_snapshot_spec import ColumnTypeSnapshotSpec from ..models.delete_stored_data_queue_job_parameters import ( DeleteStoredDataQueueJobParameters, @@ -359,6 +374,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: _data_clean_job_template ) + _advanced_properties = d.pop("advanced_properties", UNSET) + advanced_properties: Union[Unset, ColumnListModelAdvancedProperties] + if isinstance(_advanced_properties, Unset): + advanced_properties = UNSET + else: + advanced_properties = ColumnListModelAdvancedProperties.from_dict( + _advanced_properties + ) + can_edit = d.pop("can_edit", UNSET) can_collect_statistics = d.pop("can_collect_statistics", UNSET) @@ -388,6 +412,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: run_partition_checks_job_template=run_partition_checks_job_template, collect_statistics_job_template=collect_statistics_job_template, data_clean_job_template=data_clean_job_template, + advanced_properties=advanced_properties, can_edit=can_edit, can_collect_statistics=can_collect_statistics, can_run_checks=can_run_checks, diff --git a/distribution/python/dqops/client/models/column_list_model_advanced_properties.py b/distribution/python/dqops/client/models/column_list_model_advanced_properties.py new file mode 100644 index 0000000000..5aad5154ec --- /dev/null +++ b/distribution/python/dqops/client/models/column_list_model_advanced_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="ColumnListModelAdvancedProperties") + + +@_attrs_define +class ColumnListModelAdvancedProperties: + """A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value + dictionary. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + column_list_model_advanced_properties = cls() + + column_list_model_advanced_properties.additional_properties = d + return column_list_model_advanced_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/default_column_checks_pattern_list_model.py b/distribution/python/dqops/client/models/column_quality_policy_list_model.py similarity index 82% rename from distribution/python/dqops/client/models/default_column_checks_pattern_list_model.py rename to distribution/python/dqops/client/models/column_quality_policy_list_model.py index fd30de1b8a..b1c23b9d4f 100644 --- a/distribution/python/dqops/client/models/default_column_checks_pattern_list_model.py +++ b/distribution/python/dqops/client/models/column_quality_policy_list_model.py @@ -9,17 +9,17 @@ from ..models.target_column_pattern_spec import TargetColumnPatternSpec -T = TypeVar("T", bound="DefaultColumnChecksPatternListModel") +T = TypeVar("T", bound="ColumnQualityPolicyListModel") @_attrs_define -class DefaultColumnChecksPatternListModel: - """Default column-level checks pattern list model +class ColumnQualityPolicyListModel: + """Default column-level checks pattern (data quality policy) list model Attributes: - pattern_name (Union[Unset, str]): Pattern name. - priority (Union[Unset, int]): The priority of the pattern. Patterns with lower values are applied before - patterns with higher priority values. + policy_name (Union[Unset, str]): Quality policy name. + priority (Union[Unset, int]): The priority of the policy. Policies with lower values are applied before policies + with higher priority values. disabled (Union[Unset, bool]): Disables this data quality check configuration. The checks will not be activated. description (Union[Unset, str]): The description (documentation) of this data quality check configuration. target_column (Union[Unset, TargetColumnPatternSpec]): @@ -29,7 +29,7 @@ class DefaultColumnChecksPatternListModel: error message and the file location. """ - pattern_name: Union[Unset, str] = UNSET + policy_name: Union[Unset, str] = UNSET priority: Union[Unset, int] = UNSET disabled: Union[Unset, bool] = UNSET description: Union[Unset, str] = UNSET @@ -39,7 +39,7 @@ class DefaultColumnChecksPatternListModel: additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: - pattern_name = self.pattern_name + policy_name = self.policy_name priority = self.priority disabled = self.disabled description = self.description @@ -53,8 +53,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) - if pattern_name is not UNSET: - field_dict["pattern_name"] = pattern_name + if policy_name is not UNSET: + field_dict["policy_name"] = policy_name if priority is not UNSET: field_dict["priority"] = priority if disabled is not UNSET: @@ -75,7 +75,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.target_column_pattern_spec import TargetColumnPatternSpec d = src_dict.copy() - pattern_name = d.pop("pattern_name", UNSET) + policy_name = d.pop("policy_name", UNSET) priority = d.pop("priority", UNSET) @@ -94,8 +94,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: yaml_parsing_error = d.pop("yaml_parsing_error", UNSET) - default_column_checks_pattern_list_model = cls( - pattern_name=pattern_name, + column_quality_policy_list_model = cls( + policy_name=policy_name, priority=priority, disabled=disabled, description=description, @@ -104,8 +104,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: yaml_parsing_error=yaml_parsing_error, ) - default_column_checks_pattern_list_model.additional_properties = d - return default_column_checks_pattern_list_model + column_quality_policy_list_model.additional_properties = d + return column_quality_policy_list_model @property def additional_keys(self) -> List[str]: diff --git a/distribution/python/dqops/client/models/default_table_checks_pattern_model.py b/distribution/python/dqops/client/models/column_quality_policy_model.py similarity index 57% rename from distribution/python/dqops/client/models/default_table_checks_pattern_model.py rename to distribution/python/dqops/client/models/column_quality_policy_model.py index 00f81f08da..a1effa4c78 100644 --- a/distribution/python/dqops/client/models/default_table_checks_pattern_model.py +++ b/distribution/python/dqops/client/models/column_quality_policy_model.py @@ -6,36 +6,36 @@ from ..types import UNSET, Unset if TYPE_CHECKING: - from ..models.table_default_checks_pattern_spec import TableDefaultChecksPatternSpec + from ..models.column_quality_policy_spec import ColumnQualityPolicySpec -T = TypeVar("T", bound="DefaultTableChecksPatternModel") +T = TypeVar("T", bound="ColumnQualityPolicyModel") @_attrs_define -class DefaultTableChecksPatternModel: - """Default table-level checks pattern model +class ColumnQualityPolicyModel: + """Default column-level checks pattern (data quality policy) model Attributes: - pattern_name (Union[Unset, str]): Pattern name - pattern_spec (Union[Unset, TableDefaultChecksPatternSpec]): + policy_name (Union[Unset, str]): Quality policy name + policy_spec (Union[Unset, ColumnQualityPolicySpec]): can_edit (Union[Unset, bool]): Boolean flag that decides if the current user can update or delete this object. yaml_parsing_error (Union[Unset, str]): Optional parsing error that was captured when parsing the YAML file. This field is null when the YAML file is valid. If an error was captured, this field returns the file parsing error message and the file location. """ - pattern_name: Union[Unset, str] = UNSET - pattern_spec: Union[Unset, "TableDefaultChecksPatternSpec"] = UNSET + policy_name: Union[Unset, str] = UNSET + policy_spec: Union[Unset, "ColumnQualityPolicySpec"] = UNSET can_edit: Union[Unset, bool] = UNSET yaml_parsing_error: Union[Unset, str] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: - pattern_name = self.pattern_name - pattern_spec: Union[Unset, Dict[str, Any]] = UNSET - if not isinstance(self.pattern_spec, Unset): - pattern_spec = self.pattern_spec.to_dict() + policy_name = self.policy_name + policy_spec: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.policy_spec, Unset): + policy_spec = self.policy_spec.to_dict() can_edit = self.can_edit yaml_parsing_error = self.yaml_parsing_error @@ -43,10 +43,10 @@ def to_dict(self) -> Dict[str, Any]: field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) - if pattern_name is not UNSET: - field_dict["pattern_name"] = pattern_name - if pattern_spec is not UNSET: - field_dict["pattern_spec"] = pattern_spec + if policy_name is not UNSET: + field_dict["policy_name"] = policy_name + if policy_spec is not UNSET: + field_dict["policy_spec"] = policy_spec if can_edit is not UNSET: field_dict["can_edit"] = can_edit if yaml_parsing_error is not UNSET: @@ -56,33 +56,31 @@ def to_dict(self) -> Dict[str, Any]: @classmethod def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: - from ..models.table_default_checks_pattern_spec import ( - TableDefaultChecksPatternSpec, - ) + from ..models.column_quality_policy_spec import ColumnQualityPolicySpec d = src_dict.copy() - pattern_name = d.pop("pattern_name", UNSET) + policy_name = d.pop("policy_name", UNSET) - _pattern_spec = d.pop("pattern_spec", UNSET) - pattern_spec: Union[Unset, TableDefaultChecksPatternSpec] - if isinstance(_pattern_spec, Unset): - pattern_spec = UNSET + _policy_spec = d.pop("policy_spec", UNSET) + policy_spec: Union[Unset, ColumnQualityPolicySpec] + if isinstance(_policy_spec, Unset): + policy_spec = UNSET else: - pattern_spec = TableDefaultChecksPatternSpec.from_dict(_pattern_spec) + policy_spec = ColumnQualityPolicySpec.from_dict(_policy_spec) can_edit = d.pop("can_edit", UNSET) yaml_parsing_error = d.pop("yaml_parsing_error", UNSET) - default_table_checks_pattern_model = cls( - pattern_name=pattern_name, - pattern_spec=pattern_spec, + column_quality_policy_model = cls( + policy_name=policy_name, + policy_spec=policy_spec, can_edit=can_edit, yaml_parsing_error=yaml_parsing_error, ) - default_table_checks_pattern_model.additional_properties = d - return default_table_checks_pattern_model + column_quality_policy_model.additional_properties = d + return column_quality_policy_model @property def additional_keys(self) -> List[str]: diff --git a/distribution/python/dqops/client/models/column_default_checks_pattern_spec.py b/distribution/python/dqops/client/models/column_quality_policy_spec.py similarity index 96% rename from distribution/python/dqops/client/models/column_default_checks_pattern_spec.py rename to distribution/python/dqops/client/models/column_quality_policy_spec.py index 6c654dd795..9a22e1bc26 100644 --- a/distribution/python/dqops/client/models/column_default_checks_pattern_spec.py +++ b/distribution/python/dqops/client/models/column_quality_policy_spec.py @@ -18,11 +18,11 @@ from ..models.target_column_pattern_spec import TargetColumnPatternSpec -T = TypeVar("T", bound="ColumnDefaultChecksPatternSpec") +T = TypeVar("T", bound="ColumnQualityPolicySpec") @_attrs_define -class ColumnDefaultChecksPatternSpec: +class ColumnQualityPolicySpec: """ Attributes: priority (Union[Unset, int]): The priority of the pattern. Patterns with lower values are applied before @@ -138,7 +138,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: _partitioned_checks ) - column_default_checks_pattern_spec = cls( + column_quality_policy_spec = cls( priority=priority, disabled=disabled, description=description, @@ -148,8 +148,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: partitioned_checks=partitioned_checks, ) - column_default_checks_pattern_spec.additional_properties = d - return column_default_checks_pattern_spec + column_quality_policy_spec.additional_properties = d + return column_quality_policy_spec @property def additional_keys(self) -> List[str]: diff --git a/distribution/python/dqops/client/models/column_spec.py b/distribution/python/dqops/client/models/column_spec.py index 9a3dd7598f..a73c7eae35 100644 --- a/distribution/python/dqops/client/models/column_spec.py +++ b/distribution/python/dqops/client/models/column_spec.py @@ -15,6 +15,7 @@ from ..models.column_profiling_check_categories_spec import ( ColumnProfilingCheckCategoriesSpec, ) + from ..models.column_spec_advanced_properties import ColumnSpecAdvancedProperties from ..models.column_statistics_collectors_root_categories_spec import ( ColumnStatisticsCollectorsRootCategoriesSpec, ) @@ -50,6 +51,8 @@ class ColumnSpec: comments (Union[Unset, List['CommentSpec']]): Comments for change tracking. Please put comments in this collection because YAML comments may be removed when the YAML file is modified by the tool (serialization and deserialization will remove non tracked comments). + advanced_properties (Union[Unset, ColumnSpecAdvancedProperties]): A dictionary of advanced properties that can + be used for e.g. to support mapping data to data catalogs, a key/value dictionary. """ disabled: Union[Unset, bool] = UNSET @@ -62,6 +65,7 @@ class ColumnSpec: statistics: Union[Unset, "ColumnStatisticsCollectorsRootCategoriesSpec"] = UNSET labels: Union[Unset, List[str]] = UNSET comments: Union[Unset, List["CommentSpec"]] = UNSET + advanced_properties: Union[Unset, "ColumnSpecAdvancedProperties"] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -100,6 +104,10 @@ def to_dict(self) -> Dict[str, Any]: comments.append(comments_item) + advanced_properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.advanced_properties, Unset): + advanced_properties = self.advanced_properties.to_dict() + field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -123,6 +131,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["labels"] = labels if comments is not UNSET: field_dict["comments"] = comments + if advanced_properties is not UNSET: + field_dict["advanced_properties"] = advanced_properties return field_dict @@ -137,6 +147,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.column_profiling_check_categories_spec import ( ColumnProfilingCheckCategoriesSpec, ) + from ..models.column_spec_advanced_properties import ( + ColumnSpecAdvancedProperties, + ) from ..models.column_statistics_collectors_root_categories_spec import ( ColumnStatisticsCollectorsRootCategoriesSpec, ) @@ -202,6 +215,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: comments.append(comments_item) + _advanced_properties = d.pop("advanced_properties", UNSET) + advanced_properties: Union[Unset, ColumnSpecAdvancedProperties] + if isinstance(_advanced_properties, Unset): + advanced_properties = UNSET + else: + advanced_properties = ColumnSpecAdvancedProperties.from_dict( + _advanced_properties + ) + column_spec = cls( disabled=disabled, sql_expression=sql_expression, @@ -213,6 +235,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: statistics=statistics, labels=labels, comments=comments, + advanced_properties=advanced_properties, ) column_spec.additional_properties = d diff --git a/distribution/python/dqops/client/models/column_spec_advanced_properties.py b/distribution/python/dqops/client/models/column_spec_advanced_properties.py new file mode 100644 index 0000000000..91af8713ab --- /dev/null +++ b/distribution/python/dqops/client/models/column_spec_advanced_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="ColumnSpecAdvancedProperties") + + +@_attrs_define +class ColumnSpecAdvancedProperties: + """A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value + dictionary. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + column_spec_advanced_properties = cls() + + column_spec_advanced_properties.additional_properties = d + return column_spec_advanced_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/connection_model.py b/distribution/python/dqops/client/models/connection_model.py index 85e8eb94c8..74e7703db7 100644 --- a/distribution/python/dqops/client/models/connection_model.py +++ b/distribution/python/dqops/client/models/connection_model.py @@ -9,6 +9,9 @@ if TYPE_CHECKING: from ..models.big_query_parameters_spec import BigQueryParametersSpec from ..models.check_search_filters import CheckSearchFilters + from ..models.connection_model_advanced_properties import ( + ConnectionModelAdvancedProperties, + ) from ..models.databricks_parameters_spec import DatabricksParametersSpec from ..models.delete_stored_data_queue_job_parameters import ( DeleteStoredDataQueueJobParameters, @@ -33,7 +36,7 @@ @_attrs_define class ConnectionModel: - """Connection model for with a subset of parameters, excluding all nested objects. + """Connection model with a subset of parameters, excluding all nested objects. Attributes: connection_name (Union[Unset, str]): Connection name. @@ -63,6 +66,8 @@ class ConnectionModel: identifies which checks on which tables and columns should be executed. collect_statistics_job_template (Union[Unset, StatisticsCollectorSearchFilters]): data_clean_job_template (Union[Unset, DeleteStoredDataQueueJobParameters]): + advanced_properties (Union[Unset, ConnectionModelAdvancedProperties]): A dictionary of advanced properties that + can be used for e.g. to support mapping data to data catalogs, a key/value dictionary. can_edit (Union[Unset, bool]): Boolean flag that decides if the current user can update or delete the connection to the data source. can_collect_statistics (Union[Unset, bool]): Boolean flag that decides if the current user can collect @@ -98,6 +103,7 @@ class ConnectionModel: Unset, "StatisticsCollectorSearchFilters" ] = UNSET data_clean_job_template: Union[Unset, "DeleteStoredDataQueueJobParameters"] = UNSET + advanced_properties: Union[Unset, "ConnectionModelAdvancedProperties"] = UNSET can_edit: Union[Unset, bool] = UNSET can_collect_statistics: Union[Unset, bool] = UNSET can_run_checks: Union[Unset, bool] = UNSET @@ -193,6 +199,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.data_clean_job_template, Unset): data_clean_job_template = self.data_clean_job_template.to_dict() + advanced_properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.advanced_properties, Unset): + advanced_properties = self.advanced_properties.to_dict() + can_edit = self.can_edit can_collect_statistics = self.can_collect_statistics can_run_checks = self.can_run_checks @@ -254,6 +264,8 @@ def to_dict(self) -> Dict[str, Any]: ) if data_clean_job_template is not UNSET: field_dict["data_clean_job_template"] = data_clean_job_template + if advanced_properties is not UNSET: + field_dict["advanced_properties"] = advanced_properties if can_edit is not UNSET: field_dict["can_edit"] = can_edit if can_collect_statistics is not UNSET: @@ -271,6 +283,9 @@ def to_dict(self) -> Dict[str, Any]: def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.big_query_parameters_spec import BigQueryParametersSpec from ..models.check_search_filters import CheckSearchFilters + from ..models.connection_model_advanced_properties import ( + ConnectionModelAdvancedProperties, + ) from ..models.databricks_parameters_spec import DatabricksParametersSpec from ..models.delete_stored_data_queue_job_parameters import ( DeleteStoredDataQueueJobParameters, @@ -451,6 +466,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: _data_clean_job_template ) + _advanced_properties = d.pop("advanced_properties", UNSET) + advanced_properties: Union[Unset, ConnectionModelAdvancedProperties] + if isinstance(_advanced_properties, Unset): + advanced_properties = UNSET + else: + advanced_properties = ConnectionModelAdvancedProperties.from_dict( + _advanced_properties + ) + can_edit = d.pop("can_edit", UNSET) can_collect_statistics = d.pop("can_collect_statistics", UNSET) @@ -484,6 +508,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: run_partition_checks_job_template=run_partition_checks_job_template, collect_statistics_job_template=collect_statistics_job_template, data_clean_job_template=data_clean_job_template, + advanced_properties=advanced_properties, can_edit=can_edit, can_collect_statistics=can_collect_statistics, can_run_checks=can_run_checks, diff --git a/distribution/python/dqops/client/models/connection_model_advanced_properties.py b/distribution/python/dqops/client/models/connection_model_advanced_properties.py new file mode 100644 index 0000000000..c33fdf331b --- /dev/null +++ b/distribution/python/dqops/client/models/connection_model_advanced_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="ConnectionModelAdvancedProperties") + + +@_attrs_define +class ConnectionModelAdvancedProperties: + """A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value + dictionary. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + connection_model_advanced_properties = cls() + + connection_model_advanced_properties.additional_properties = d + return connection_model_advanced_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/connection_spec.py b/distribution/python/dqops/client/models/connection_spec.py index 66c41f77f1..a1b1a0fc0d 100644 --- a/distribution/python/dqops/client/models/connection_spec.py +++ b/distribution/python/dqops/client/models/connection_spec.py @@ -12,6 +12,9 @@ from ..models.connection_incident_grouping_spec import ( ConnectionIncidentGroupingSpec, ) + from ..models.connection_spec_advanced_properties import ( + ConnectionSpecAdvancedProperties, + ) from ..models.data_grouping_configuration_spec import DataGroupingConfigurationSpec from ..models.databricks_parameters_spec import DatabricksParametersSpec from ..models.default_schedules_spec import DefaultSchedulesSpec @@ -57,6 +60,8 @@ class ConnectionSpec: deserialization will remove non tracked comments). labels (Union[Unset, List[str]]): Custom labels that were assigned to the connection. Labels are used for searching for tables when filtered data quality checks are executed. + advanced_properties (Union[Unset, ConnectionSpecAdvancedProperties]): A dictionary of advanced properties that + can be used for e.g. to support mapping data to data catalogs, a key/value dictionary. """ provider_type: Union[Unset, ProviderType] = UNSET @@ -80,6 +85,7 @@ class ConnectionSpec: incident_grouping: Union[Unset, "ConnectionIncidentGroupingSpec"] = UNSET comments: Union[Unset, List["CommentSpec"]] = UNSET labels: Union[Unset, List[str]] = UNSET + advanced_properties: Union[Unset, "ConnectionSpecAdvancedProperties"] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -162,6 +168,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.labels, Unset): labels = self.labels + advanced_properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.advanced_properties, Unset): + advanced_properties = self.advanced_properties.to_dict() + field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -205,6 +215,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["comments"] = comments if labels is not UNSET: field_dict["labels"] = labels + if advanced_properties is not UNSET: + field_dict["advanced_properties"] = advanced_properties return field_dict @@ -215,6 +227,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.connection_incident_grouping_spec import ( ConnectionIncidentGroupingSpec, ) + from ..models.connection_spec_advanced_properties import ( + ConnectionSpecAdvancedProperties, + ) from ..models.data_grouping_configuration_spec import ( DataGroupingConfigurationSpec, ) @@ -359,6 +374,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: labels = cast(List[str], d.pop("labels", UNSET)) + _advanced_properties = d.pop("advanced_properties", UNSET) + advanced_properties: Union[Unset, ConnectionSpecAdvancedProperties] + if isinstance(_advanced_properties, Unset): + advanced_properties = UNSET + else: + advanced_properties = ConnectionSpecAdvancedProperties.from_dict( + _advanced_properties + ) + connection_spec = cls( provider_type=provider_type, bigquery=bigquery, @@ -379,6 +403,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: incident_grouping=incident_grouping, comments=comments, labels=labels, + advanced_properties=advanced_properties, ) connection_spec.additional_properties = d diff --git a/distribution/python/dqops/client/models/connection_spec_advanced_properties.py b/distribution/python/dqops/client/models/connection_spec_advanced_properties.py new file mode 100644 index 0000000000..acdd615709 --- /dev/null +++ b/distribution/python/dqops/client/models/connection_spec_advanced_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="ConnectionSpecAdvancedProperties") + + +@_attrs_define +class ConnectionSpecAdvancedProperties: + """A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value + dictionary. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + connection_spec_advanced_properties = cls() + + connection_spec_advanced_properties.additional_properties = d + return connection_spec_advanced_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/delete_stored_data_queue_job_parameters.py b/distribution/python/dqops/client/models/delete_stored_data_queue_job_parameters.py index fa22e59b41..54b2e13b95 100644 --- a/distribution/python/dqops/client/models/delete_stored_data_queue_job_parameters.py +++ b/distribution/python/dqops/client/models/delete_stored_data_queue_job_parameters.py @@ -37,7 +37,10 @@ class DeleteStoredDataQueueJobParameters: parameter must be set to *true* to delete the error samples. delete_incidents (Union[Unset, bool]): Delete the data from the [incidents](../../reference/parquetfiles/incidents.md) table. Because the default value is *false*, this - parameter must be set to *true* to delete the error samples. + parameter must be set to *true* to delete the incidents. + delete_checks_configuration (Union[Unset, bool]): Delete the data quality configured checks from the table. They + are detached from the configuration.Because the default value is *false*, this parameter must be set to *true* + to delete the checks configuration. column_names (Union[Unset, List[str]]): The list of column names to delete the data for column level results or errors only for selected columns. check_category (Union[Unset, str]): The check category name, for example *volume* or *anomaly*. @@ -76,6 +79,7 @@ class DeleteStoredDataQueueJobParameters: delete_sensor_readouts: Union[Unset, bool] = UNSET delete_error_samples: Union[Unset, bool] = UNSET delete_incidents: Union[Unset, bool] = UNSET + delete_checks_configuration: Union[Unset, bool] = UNSET column_names: Union[Unset, List[str]] = UNSET check_category: Union[Unset, str] = UNSET table_comparison_name: Union[Unset, str] = UNSET @@ -108,6 +112,7 @@ def to_dict(self) -> Dict[str, Any]: delete_sensor_readouts = self.delete_sensor_readouts delete_error_samples = self.delete_error_samples delete_incidents = self.delete_incidents + delete_checks_configuration = self.delete_checks_configuration column_names: Union[Unset, List[str]] = UNSET if not isinstance(self.column_names, Unset): column_names = self.column_names @@ -148,6 +153,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["deleteErrorSamples"] = delete_error_samples if delete_incidents is not UNSET: field_dict["deleteIncidents"] = delete_incidents + if delete_checks_configuration is not UNSET: + field_dict["deleteChecksConfiguration"] = delete_checks_configuration if column_names is not UNSET: field_dict["columnNames"] = column_names if check_category is not UNSET: @@ -210,6 +217,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: delete_incidents = d.pop("deleteIncidents", UNSET) + delete_checks_configuration = d.pop("deleteChecksConfiguration", UNSET) + column_names = cast(List[str], d.pop("columnNames", UNSET)) check_category = d.pop("checkCategory", UNSET) @@ -247,6 +256,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: delete_sensor_readouts=delete_sensor_readouts, delete_error_samples=delete_error_samples, delete_incidents=delete_incidents, + delete_checks_configuration=delete_checks_configuration, column_names=column_names, check_category=check_category, table_comparison_name=table_comparison_name, diff --git a/distribution/python/dqops/client/models/display_hint.py b/distribution/python/dqops/client/models/display_hint.py index 27f9d02af6..5892d2d88c 100644 --- a/distribution/python/dqops/client/models/display_hint.py +++ b/distribution/python/dqops/client/models/display_hint.py @@ -2,6 +2,7 @@ class DisplayHint(str, Enum): + COLUMN_NAMES = "column_names" TEXTAREA = "textarea" def __str__(self) -> str: diff --git a/distribution/python/dqops/client/models/dqo_job_change_model.py b/distribution/python/dqops/client/models/dqo_job_change_model.py index 928f4ff4f8..a1cbec4a89 100644 --- a/distribution/python/dqops/client/models/dqo_job_change_model.py +++ b/distribution/python/dqops/client/models/dqo_job_change_model.py @@ -23,6 +23,7 @@ class DqoJobChangeModel: change_sequence (Union[Unset, int]): updated_model (Union[Unset, DqoJobHistoryEntryModel]): status_changed_at (Union[Unset, int]): + domain_name (Union[Unset, str]): """ status: Union[Unset, DqoJobStatus] = UNSET @@ -30,6 +31,7 @@ class DqoJobChangeModel: change_sequence: Union[Unset, int] = UNSET updated_model: Union[Unset, "DqoJobHistoryEntryModel"] = UNSET status_changed_at: Union[Unset, int] = UNSET + domain_name: Union[Unset, str] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -47,6 +49,7 @@ def to_dict(self) -> Dict[str, Any]: updated_model = self.updated_model.to_dict() status_changed_at = self.status_changed_at + domain_name = self.domain_name field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) @@ -61,6 +64,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["updatedModel"] = updated_model if status_changed_at is not UNSET: field_dict["statusChangedAt"] = status_changed_at + if domain_name is not UNSET: + field_dict["domainName"] = domain_name return field_dict @@ -95,12 +100,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: status_changed_at = d.pop("statusChangedAt", UNSET) + domain_name = d.pop("domainName", UNSET) + dqo_job_change_model = cls( status=status, job_id=job_id, change_sequence=change_sequence, updated_model=updated_model, status_changed_at=status_changed_at, + domain_name=domain_name, ) dqo_job_change_model.additional_properties = d diff --git a/distribution/python/dqops/client/models/dqo_job_history_entry_model.py b/distribution/python/dqops/client/models/dqo_job_history_entry_model.py index f22d6cc0d2..1de64fc904 100644 --- a/distribution/python/dqops/client/models/dqo_job_history_entry_model.py +++ b/distribution/python/dqops/client/models/dqo_job_history_entry_model.py @@ -25,6 +25,7 @@ class DqoJobHistoryEntryModel: status (Union[Unset, DqoJobStatus]): error_message (Union[Unset, str]): status_changed_at (Union[Unset, int]): + data_domain (Union[Unset, str]): """ job_id: Union[Unset, "DqoQueueJobId"] = UNSET @@ -33,6 +34,7 @@ class DqoJobHistoryEntryModel: status: Union[Unset, DqoJobStatus] = UNSET error_message: Union[Unset, str] = UNSET status_changed_at: Union[Unset, int] = UNSET + data_domain: Union[Unset, str] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -54,6 +56,7 @@ def to_dict(self) -> Dict[str, Any]: error_message = self.error_message status_changed_at = self.status_changed_at + data_domain = self.data_domain field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) @@ -70,6 +73,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["errorMessage"] = error_message if status_changed_at is not UNSET: field_dict["statusChangedAt"] = status_changed_at + if data_domain is not UNSET: + field_dict["dataDomain"] = data_domain return field_dict @@ -111,6 +116,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: status_changed_at = d.pop("statusChangedAt", UNSET) + data_domain = d.pop("dataDomain", UNSET) + dqo_job_history_entry_model = cls( job_id=job_id, job_type=job_type, @@ -118,6 +125,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: status=status, error_message=error_message, status_changed_at=status_changed_at, + data_domain=data_domain, ) dqo_job_history_entry_model.additional_properties = d diff --git a/distribution/python/dqops/client/models/dqo_user_profile_model.py b/distribution/python/dqops/client/models/dqo_user_profile_model.py index fa378e1af7..b9ad759af0 100644 --- a/distribution/python/dqops/client/models/dqo_user_profile_model.py +++ b/distribution/python/dqops/client/models/dqo_user_profile_model.py @@ -56,6 +56,8 @@ class DqoUserProfileModel: download) already defined shared credentials. can_change_own_password (Union[Unset, bool]): User can change his own password in DQOps Cloud, because the DQOps Cloud Pairing API Key is valid and synchronization is enabled. + can_use_data_domains (Union[Unset, bool]): User can use data domains. Support for data domains requires an + ENTERPRISE license of DQOps. """ user: Union[Unset, str] = UNSET @@ -86,6 +88,7 @@ class DqoUserProfileModel: can_manage_users: Union[Unset, bool] = UNSET can_manage_and_view_shared_credentials: Union[Unset, bool] = UNSET can_change_own_password: Union[Unset, bool] = UNSET + can_use_data_domains: Union[Unset, bool] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -122,6 +125,7 @@ def to_dict(self) -> Dict[str, Any]: self.can_manage_and_view_shared_credentials ) can_change_own_password = self.can_change_own_password + can_use_data_domains = self.can_use_data_domains field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) @@ -186,6 +190,8 @@ def to_dict(self) -> Dict[str, Any]: ) if can_change_own_password is not UNSET: field_dict["can_change_own_password"] = can_change_own_password + if can_use_data_domains is not UNSET: + field_dict["can_use_data_domains"] = can_use_data_domains return field_dict @@ -257,6 +263,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: can_change_own_password = d.pop("can_change_own_password", UNSET) + can_use_data_domains = d.pop("can_use_data_domains", UNSET) + dqo_user_profile_model = cls( user=user, tenant=tenant, @@ -286,6 +294,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: can_manage_users=can_manage_users, can_manage_and_view_shared_credentials=can_manage_and_view_shared_credentials, can_change_own_password=can_change_own_password, + can_use_data_domains=can_use_data_domains, ) dqo_user_profile_model.additional_properties = d diff --git a/distribution/python/dqops/client/models/dqo_cloud_user_model.py b/distribution/python/dqops/client/models/dqo_user_roles_model.py similarity index 54% rename from distribution/python/dqops/client/models/dqo_cloud_user_model.py rename to distribution/python/dqops/client/models/dqo_user_roles_model.py index 6fea834581..e692818b1a 100644 --- a/distribution/python/dqops/client/models/dqo_cloud_user_model.py +++ b/distribution/python/dqops/client/models/dqo_user_roles_model.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union from attrs import define as _attrs_define from attrs import field as _attrs_field @@ -6,20 +6,30 @@ from ..models.dqo_user_role import DqoUserRole from ..types import UNSET, Unset -T = TypeVar("T", bound="DqoCloudUserModel") +if TYPE_CHECKING: + from ..models.dqo_user_roles_model_data_domain_roles import ( + DqoUserRolesModelDataDomainRoles, + ) + + +T = TypeVar("T", bound="DqoUserRolesModel") @_attrs_define -class DqoCloudUserModel: - """DQOps Cloud user model - identifies a user in a multi-user DQOps deployment. +class DqoUserRolesModel: + """DQOps user model - identifies a user in a multi-user DQOps deployment and the user's roles. Attributes: email (Union[Unset, str]): User's email that identifies the user. account_role (Union[Unset, DqoUserRole]): + data_domain_roles (Union[Unset, DqoUserRolesModelDataDomainRoles]): User roles within each data domain. Data + domains are supported in an ENTERPRISE version of DQOps and they are managed by the SaaS components of DQOps + Cloud. """ email: Union[Unset, str] = UNSET account_role: Union[Unset, DqoUserRole] = UNSET + data_domain_roles: Union[Unset, "DqoUserRolesModelDataDomainRoles"] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -28,6 +38,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.account_role, Unset): account_role = self.account_role.value + data_domain_roles: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.data_domain_roles, Unset): + data_domain_roles = self.data_domain_roles.to_dict() + field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -35,11 +49,17 @@ def to_dict(self) -> Dict[str, Any]: field_dict["email"] = email if account_role is not UNSET: field_dict["accountRole"] = account_role + if data_domain_roles is not UNSET: + field_dict["dataDomainRoles"] = data_domain_roles return field_dict @classmethod def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.dqo_user_roles_model_data_domain_roles import ( + DqoUserRolesModelDataDomainRoles, + ) + d = src_dict.copy() email = d.pop("email", UNSET) @@ -50,13 +70,23 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: account_role = DqoUserRole(_account_role) - dqo_cloud_user_model = cls( + _data_domain_roles = d.pop("dataDomainRoles", UNSET) + data_domain_roles: Union[Unset, DqoUserRolesModelDataDomainRoles] + if isinstance(_data_domain_roles, Unset): + data_domain_roles = UNSET + else: + data_domain_roles = DqoUserRolesModelDataDomainRoles.from_dict( + _data_domain_roles + ) + + dqo_user_roles_model = cls( email=email, account_role=account_role, + data_domain_roles=data_domain_roles, ) - dqo_cloud_user_model.additional_properties = d - return dqo_cloud_user_model + dqo_user_roles_model.additional_properties = d + return dqo_user_roles_model @property def additional_keys(self) -> List[str]: diff --git a/distribution/python/dqops/client/models/dqo_user_roles_model_data_domain_roles.py b/distribution/python/dqops/client/models/dqo_user_roles_model_data_domain_roles.py new file mode 100644 index 0000000000..86008196c9 --- /dev/null +++ b/distribution/python/dqops/client/models/dqo_user_roles_model_data_domain_roles.py @@ -0,0 +1,62 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..models.dqo_user_role import DqoUserRole + +T = TypeVar("T", bound="DqoUserRolesModelDataDomainRoles") + + +@_attrs_define +class DqoUserRolesModelDataDomainRoles: + """User roles within each data domain. Data domains are supported in an ENTERPRISE version of DQOps and they are + managed by the SaaS components of DQOps Cloud. + + """ + + additional_properties: Dict[str, DqoUserRole] = _attrs_field( + init=False, factory=dict + ) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.value + + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + dqo_user_roles_model_data_domain_roles = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = DqoUserRole(prop_dict) + + additional_properties[prop_name] = additional_property + + dqo_user_roles_model_data_domain_roles.additional_properties = ( + additional_properties + ) + return dqo_user_roles_model_data_domain_roles + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> DqoUserRole: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: DqoUserRole) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/local_data_domain_model.py b/distribution/python/dqops/client/models/local_data_domain_model.py new file mode 100644 index 0000000000..69b65488b4 --- /dev/null +++ b/distribution/python/dqops/client/models/local_data_domain_model.py @@ -0,0 +1,93 @@ +import datetime +from typing import Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from dateutil.parser import isoparse + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="LocalDataDomainModel") + + +@_attrs_define +class LocalDataDomainModel: + """Model that describes a local data domain. It is also used to create new data domains. + + Attributes: + domain_name (Union[Unset, str]): Data domain name. + display_name (Union[Unset, str]): Data domain display name (user-friendly name). + created_at (Union[Unset, datetime.datetime]): Data domain creation time. + enable_scheduler (Union[Unset, bool]): Enables the job scheduler for this domain. + """ + + domain_name: Union[Unset, str] = UNSET + display_name: Union[Unset, str] = UNSET + created_at: Union[Unset, datetime.datetime] = UNSET + enable_scheduler: Union[Unset, bool] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + domain_name = self.domain_name + display_name = self.display_name + created_at: Union[Unset, str] = UNSET + if not isinstance(self.created_at, Unset): + created_at = self.created_at.isoformat() + + enable_scheduler = self.enable_scheduler + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if domain_name is not UNSET: + field_dict["domain_name"] = domain_name + if display_name is not UNSET: + field_dict["display_name"] = display_name + if created_at is not UNSET: + field_dict["created_at"] = created_at + if enable_scheduler is not UNSET: + field_dict["enable_scheduler"] = enable_scheduler + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + domain_name = d.pop("domain_name", UNSET) + + display_name = d.pop("display_name", UNSET) + + _created_at = d.pop("created_at", UNSET) + created_at: Union[Unset, datetime.datetime] + if isinstance(_created_at, Unset): + created_at = UNSET + else: + created_at = isoparse(_created_at) + + enable_scheduler = d.pop("enable_scheduler", UNSET) + + local_data_domain_model = cls( + domain_name=domain_name, + display_name=display_name, + created_at=created_at, + enable_scheduler=enable_scheduler, + ) + + local_data_domain_model.additional_properties = d + return local_data_domain_model + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/notification_filter_spec.py b/distribution/python/dqops/client/models/notification_filter_spec.py index 65fb398947..1c646c4b4c 100644 --- a/distribution/python/dqops/client/models/notification_filter_spec.py +++ b/distribution/python/dqops/client/models/notification_filter_spec.py @@ -3,6 +3,7 @@ from attrs import define as _attrs_define from attrs import field as _attrs_field +from ..models.check_type import CheckType from ..types import UNSET, Unset T = TypeVar("T", bound="NotificationFilterSpec") @@ -23,8 +24,7 @@ class NotificationFilterSpec: 'group_name_\*', '\*group', 'prefix\*suffix'. quality_dimension (Union[Unset, str]): Quality dimension. check_category (Union[Unset, str]): The target check category, for example: *nulls*, *volume*, *anomaly*. - check_type (Union[Unset, str]): The target type of checks to run. Supported values are *profiling*, *monitoring* - and *partitioned*. + check_type (Union[Unset, CheckType]): check_name (Union[Unset, str]): The target check name to run only this named check. Uses the short check name which is the name of the deepest folder in the *checks* folder. This field supports search patterns such as: 'profiling_\*', '\*_count', 'profiling_\*_percent'. @@ -38,7 +38,7 @@ class NotificationFilterSpec: data_group_name: Union[Unset, str] = UNSET quality_dimension: Union[Unset, str] = UNSET check_category: Union[Unset, str] = UNSET - check_type: Union[Unset, str] = UNSET + check_type: Union[Unset, CheckType] = UNSET check_name: Union[Unset, str] = UNSET highest_severity: Union[Unset, int] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) @@ -51,7 +51,10 @@ def to_dict(self) -> Dict[str, Any]: data_group_name = self.data_group_name quality_dimension = self.quality_dimension check_category = self.check_category - check_type = self.check_type + check_type: Union[Unset, str] = UNSET + if not isinstance(self.check_type, Unset): + check_type = self.check_type.value + check_name = self.check_name highest_severity = self.highest_severity @@ -98,7 +101,12 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: check_category = d.pop("checkCategory", UNSET) - check_type = d.pop("checkType", UNSET) + _check_type = d.pop("checkType", UNSET) + check_type: Union[Unset, CheckType] + if isinstance(_check_type, Unset): + check_type = UNSET + else: + check_type = CheckType(_check_type) check_name = d.pop("checkName", UNSET) diff --git a/distribution/python/dqops/client/models/statistics_collector_search_filters.py b/distribution/python/dqops/client/models/statistics_collector_search_filters.py index bf5adfb466..f9725d75a6 100644 --- a/distribution/python/dqops/client/models/statistics_collector_search_filters.py +++ b/distribution/python/dqops/client/models/statistics_collector_search_filters.py @@ -45,7 +45,6 @@ class StatisticsCollectorSearchFilters: collector_category (Union[Unset, str]): The target statistics collector category, for example: *nulls*, *volume*, *sampling*. target (Union[Unset, StatisticsCollectorTarget]): - samples_limit (Union[Unset, int]): The default limit of column samples that are collected. collectors_hierarchy_ids_models (Union[Unset, List['HierarchyIdModel']]): """ @@ -60,7 +59,6 @@ class StatisticsCollectorSearchFilters: sensor_name: Union[Unset, str] = UNSET collector_category: Union[Unset, str] = UNSET target: Union[Unset, StatisticsCollectorTarget] = UNSET - samples_limit: Union[Unset, int] = UNSET collectors_hierarchy_ids_models: Union[Unset, List["HierarchyIdModel"]] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) @@ -88,7 +86,6 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.target, Unset): target = self.target.value - samples_limit = self.samples_limit collectors_hierarchy_ids_models: Union[Unset, List[Dict[str, Any]]] = UNSET if not isinstance(self.collectors_hierarchy_ids_models, Unset): collectors_hierarchy_ids_models = [] @@ -128,8 +125,6 @@ def to_dict(self) -> Dict[str, Any]: field_dict["collectorCategory"] = collector_category if target is not UNSET: field_dict["target"] = target - if samples_limit is not UNSET: - field_dict["samplesLimit"] = samples_limit if collectors_hierarchy_ids_models is not UNSET: field_dict["collectorsHierarchyIdsModels"] = collectors_hierarchy_ids_models @@ -167,8 +162,6 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: target = StatisticsCollectorTarget(_target) - samples_limit = d.pop("samplesLimit", UNSET) - collectors_hierarchy_ids_models = [] _collectors_hierarchy_ids_models = d.pop("collectorsHierarchyIdsModels", UNSET) for collectors_hierarchy_ids_models_item_data in ( @@ -192,7 +185,6 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: sensor_name=sensor_name, collector_category=collector_category, target=target, - samples_limit=samples_limit, collectors_hierarchy_ids_models=collectors_hierarchy_ids_models, ) diff --git a/distribution/python/dqops/client/models/table_current_data_quality_status_model.py b/distribution/python/dqops/client/models/table_current_data_quality_status_model.py index 11486d847f..e96cb70730 100644 --- a/distribution/python/dqops/client/models/table_current_data_quality_status_model.py +++ b/distribution/python/dqops/client/models/table_current_data_quality_status_model.py @@ -60,6 +60,8 @@ class TableCurrentDataQualityStatusModel: that have any known data quality results. The keys in the dictionary are the column names. dimensions (Union[Unset, TableCurrentDataQualityStatusModelDimensions]): Dictionary of the current data quality statues for each data quality dimension. + table_exist (Union[Unset, bool]): The flag informing whether the table exist. The table might not exist for the + imported data lineage source tables. """ connection_name: Union[Unset, str] = UNSET @@ -78,6 +80,7 @@ class TableCurrentDataQualityStatusModel: checks: Union[Unset, "TableCurrentDataQualityStatusModelChecks"] = UNSET columns: Union[Unset, "TableCurrentDataQualityStatusModelColumns"] = UNSET dimensions: Union[Unset, "TableCurrentDataQualityStatusModelDimensions"] = UNSET + table_exist: Union[Unset, bool] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -112,6 +115,8 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.dimensions, Unset): dimensions = self.dimensions.to_dict() + table_exist = self.table_exist + field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -147,6 +152,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["columns"] = columns if dimensions is not UNSET: field_dict["dimensions"] = dimensions + if table_exist is not UNSET: + field_dict["table_exist"] = table_exist return field_dict @@ -224,6 +231,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: _dimensions ) + table_exist = d.pop("table_exist", UNSET) + table_current_data_quality_status_model = cls( connection_name=connection_name, schema_name=schema_name, @@ -241,6 +250,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: checks=checks, columns=columns, dimensions=dimensions, + table_exist=table_exist, ) table_current_data_quality_status_model.additional_properties = d diff --git a/distribution/python/dqops/client/models/table_daily_monitoring_check_categories_spec.py b/distribution/python/dqops/client/models/table_daily_monitoring_check_categories_spec.py index 868fb35f20..14063044d9 100644 --- a/distribution/python/dqops/client/models/table_daily_monitoring_check_categories_spec.py +++ b/distribution/python/dqops/client/models/table_daily_monitoring_check_categories_spec.py @@ -27,6 +27,9 @@ from ..models.table_timeliness_daily_monitoring_checks_spec import ( TableTimelinessDailyMonitoringChecksSpec, ) + from ..models.table_uniqueness_daily_monitoring_checks_spec import ( + TableUniquenessDailyMonitoringChecksSpec, + ) from ..models.table_volume_daily_monitoring_checks_spec import ( TableVolumeDailyMonitoringChecksSpec, ) @@ -47,6 +50,7 @@ class TableDailyMonitoringCheckCategoriesSpec: custom_sql (Union[Unset, TableCustomSqlDailyMonitoringChecksSpec]): availability (Union[Unset, TableAvailabilityDailyMonitoringChecksSpec]): schema (Union[Unset, TableSchemaDailyMonitoringChecksSpec]): + uniqueness (Union[Unset, TableUniquenessDailyMonitoringChecksSpec]): comparisons (Union[Unset, TableDailyMonitoringCheckCategoriesSpecComparisons]): Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table. @@ -59,6 +63,7 @@ class TableDailyMonitoringCheckCategoriesSpec: custom_sql: Union[Unset, "TableCustomSqlDailyMonitoringChecksSpec"] = UNSET availability: Union[Unset, "TableAvailabilityDailyMonitoringChecksSpec"] = UNSET schema: Union[Unset, "TableSchemaDailyMonitoringChecksSpec"] = UNSET + uniqueness: Union[Unset, "TableUniquenessDailyMonitoringChecksSpec"] = UNSET comparisons: Union[Unset, "TableDailyMonitoringCheckCategoriesSpecComparisons"] = ( UNSET ) @@ -93,6 +98,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.schema, Unset): schema = self.schema.to_dict() + uniqueness: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.uniqueness, Unset): + uniqueness = self.uniqueness.to_dict() + comparisons: Union[Unset, Dict[str, Any]] = UNSET if not isinstance(self.comparisons, Unset): comparisons = self.comparisons.to_dict() @@ -114,6 +123,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["availability"] = availability if schema is not UNSET: field_dict["schema"] = schema + if uniqueness is not UNSET: + field_dict["uniqueness"] = uniqueness if comparisons is not UNSET: field_dict["comparisons"] = comparisons @@ -142,6 +153,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.table_timeliness_daily_monitoring_checks_spec import ( TableTimelinessDailyMonitoringChecksSpec, ) + from ..models.table_uniqueness_daily_monitoring_checks_spec import ( + TableUniquenessDailyMonitoringChecksSpec, + ) from ..models.table_volume_daily_monitoring_checks_spec import ( TableVolumeDailyMonitoringChecksSpec, ) @@ -198,6 +212,13 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: schema = TableSchemaDailyMonitoringChecksSpec.from_dict(_schema) + _uniqueness = d.pop("uniqueness", UNSET) + uniqueness: Union[Unset, TableUniquenessDailyMonitoringChecksSpec] + if isinstance(_uniqueness, Unset): + uniqueness = UNSET + else: + uniqueness = TableUniquenessDailyMonitoringChecksSpec.from_dict(_uniqueness) + _comparisons = d.pop("comparisons", UNSET) comparisons: Union[Unset, TableDailyMonitoringCheckCategoriesSpecComparisons] if isinstance(_comparisons, Unset): @@ -215,6 +236,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: custom_sql=custom_sql, availability=availability, schema=schema, + uniqueness=uniqueness, comparisons=comparisons, ) diff --git a/distribution/python/dqops/client/models/table_daily_partitioned_check_categories_spec.py b/distribution/python/dqops/client/models/table_daily_partitioned_check_categories_spec.py index fc3db731aa..2bcdc5111d 100644 --- a/distribution/python/dqops/client/models/table_daily_partitioned_check_categories_spec.py +++ b/distribution/python/dqops/client/models/table_daily_partitioned_check_categories_spec.py @@ -18,6 +18,9 @@ from ..models.table_timeliness_daily_partitioned_checks_spec import ( TableTimelinessDailyPartitionedChecksSpec, ) + from ..models.table_uniqueness_daily_partition_checks_spec import ( + TableUniquenessDailyPartitionChecksSpec, + ) from ..models.table_volume_daily_partitioned_checks_spec import ( TableVolumeDailyPartitionedChecksSpec, ) @@ -35,6 +38,7 @@ class TableDailyPartitionedCheckCategoriesSpec: volume (Union[Unset, TableVolumeDailyPartitionedChecksSpec]): timeliness (Union[Unset, TableTimelinessDailyPartitionedChecksSpec]): custom_sql (Union[Unset, TableCustomSqlDailyPartitionedChecksSpec]): + uniqueness (Union[Unset, TableUniquenessDailyPartitionChecksSpec]): comparisons (Union[Unset, TableDailyPartitionedCheckCategoriesSpecComparisons]): Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table. @@ -44,6 +48,7 @@ class TableDailyPartitionedCheckCategoriesSpec: volume: Union[Unset, "TableVolumeDailyPartitionedChecksSpec"] = UNSET timeliness: Union[Unset, "TableTimelinessDailyPartitionedChecksSpec"] = UNSET custom_sql: Union[Unset, "TableCustomSqlDailyPartitionedChecksSpec"] = UNSET + uniqueness: Union[Unset, "TableUniquenessDailyPartitionChecksSpec"] = UNSET comparisons: Union[Unset, "TableDailyPartitionedCheckCategoriesSpecComparisons"] = ( UNSET ) @@ -66,6 +71,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.custom_sql, Unset): custom_sql = self.custom_sql.to_dict() + uniqueness: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.uniqueness, Unset): + uniqueness = self.uniqueness.to_dict() + comparisons: Union[Unset, Dict[str, Any]] = UNSET if not isinstance(self.comparisons, Unset): comparisons = self.comparisons.to_dict() @@ -81,6 +90,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["timeliness"] = timeliness if custom_sql is not UNSET: field_dict["custom_sql"] = custom_sql + if uniqueness is not UNSET: + field_dict["uniqueness"] = uniqueness if comparisons is not UNSET: field_dict["comparisons"] = comparisons @@ -100,6 +111,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.table_timeliness_daily_partitioned_checks_spec import ( TableTimelinessDailyPartitionedChecksSpec, ) + from ..models.table_uniqueness_daily_partition_checks_spec import ( + TableUniquenessDailyPartitionChecksSpec, + ) from ..models.table_volume_daily_partitioned_checks_spec import ( TableVolumeDailyPartitionedChecksSpec, ) @@ -135,6 +149,13 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: custom_sql = TableCustomSqlDailyPartitionedChecksSpec.from_dict(_custom_sql) + _uniqueness = d.pop("uniqueness", UNSET) + uniqueness: Union[Unset, TableUniquenessDailyPartitionChecksSpec] + if isinstance(_uniqueness, Unset): + uniqueness = UNSET + else: + uniqueness = TableUniquenessDailyPartitionChecksSpec.from_dict(_uniqueness) + _comparisons = d.pop("comparisons", UNSET) comparisons: Union[Unset, TableDailyPartitionedCheckCategoriesSpecComparisons] if isinstance(_comparisons, Unset): @@ -149,6 +170,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: volume=volume, timeliness=timeliness, custom_sql=custom_sql, + uniqueness=uniqueness, comparisons=comparisons, ) diff --git a/distribution/python/dqops/client/models/table_duplicate_record_count_check_spec.py b/distribution/python/dqops/client/models/table_duplicate_record_count_check_spec.py new file mode 100644 index 0000000000..b509a2da3d --- /dev/null +++ b/distribution/python/dqops/client/models/table_duplicate_record_count_check_spec.py @@ -0,0 +1,254 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.comment_spec import CommentSpec + from ..models.max_count_rule_0_error_parameters_spec import ( + MaxCountRule0ErrorParametersSpec, + ) + from ..models.max_count_rule_0_warning_parameters_spec import ( + MaxCountRule0WarningParametersSpec, + ) + from ..models.max_count_rule_100_parameters_spec import ( + MaxCountRule100ParametersSpec, + ) + from ..models.monitoring_schedule_spec import MonitoringScheduleSpec + from ..models.table_duplicate_record_count_sensor_parameters_spec import ( + TableDuplicateRecordCountSensorParametersSpec, + ) + + +T = TypeVar("T", bound="TableDuplicateRecordCountCheckSpec") + + +@_attrs_define +class TableDuplicateRecordCountCheckSpec: + """ + Attributes: + schedule_override (Union[Unset, MonitoringScheduleSpec]): + comments (Union[Unset, List['CommentSpec']]): Comments for change tracking. Please put comments in this + collection because YAML comments may be removed when the YAML file is modified by the tool (serialization and + deserialization will remove non tracked comments). + disabled (Union[Unset, bool]): Disables the data quality check. Only enabled data quality checks and monitorings + are executed. The check should be disabled if it should not work, but the configuration of the sensor and rules + should be preserved in the configuration. + exclude_from_kpi (Union[Unset, bool]): Data quality check results (alerts) are included in the data quality KPI + calculation by default. Set this field to true in order to exclude this data quality check from the data quality + KPI calculation. + include_in_sla (Union[Unset, bool]): Marks the data quality check as part of a data quality SLA (Data Contract). + The data quality SLA is a set of critical data quality checks that must always pass and are considered as a Data + Contract for the dataset. + quality_dimension (Union[Unset, str]): Configures a custom data quality dimension name that is different than + the built-in dimensions (Timeliness, Validity, etc.). + display_name (Union[Unset, str]): Data quality check display name that can be assigned to the check, otherwise + the check_display_name stored in the parquet result files is the check_name. + data_grouping (Union[Unset, str]): Data grouping configuration name that should be applied to this data quality + check. The data grouping is used to group the check's result by a GROUP BY clause in SQL, evaluating the data + quality check for each group of rows. Use the name of one of data grouping configurations defined on the parent + table. + always_collect_error_samples (Union[Unset, bool]): Forces collecting error samples for this check whenever it + fails, even if it is a monitoring check that is run by a scheduler, and running an additional query to collect + error samples will impose additional load on the data source. + parameters (Union[Unset, TableDuplicateRecordCountSensorParametersSpec]): + warning (Union[Unset, MaxCountRule0WarningParametersSpec]): + error (Union[Unset, MaxCountRule0ErrorParametersSpec]): + fatal (Union[Unset, MaxCountRule100ParametersSpec]): + """ + + schedule_override: Union[Unset, "MonitoringScheduleSpec"] = UNSET + comments: Union[Unset, List["CommentSpec"]] = UNSET + disabled: Union[Unset, bool] = UNSET + exclude_from_kpi: Union[Unset, bool] = UNSET + include_in_sla: Union[Unset, bool] = UNSET + quality_dimension: Union[Unset, str] = UNSET + display_name: Union[Unset, str] = UNSET + data_grouping: Union[Unset, str] = UNSET + always_collect_error_samples: Union[Unset, bool] = UNSET + parameters: Union[Unset, "TableDuplicateRecordCountSensorParametersSpec"] = UNSET + warning: Union[Unset, "MaxCountRule0WarningParametersSpec"] = UNSET + error: Union[Unset, "MaxCountRule0ErrorParametersSpec"] = UNSET + fatal: Union[Unset, "MaxCountRule100ParametersSpec"] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + schedule_override: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.schedule_override, Unset): + schedule_override = self.schedule_override.to_dict() + + comments: Union[Unset, List[Dict[str, Any]]] = UNSET + if not isinstance(self.comments, Unset): + comments = [] + for comments_item_data in self.comments: + comments_item = comments_item_data.to_dict() + + comments.append(comments_item) + + disabled = self.disabled + exclude_from_kpi = self.exclude_from_kpi + include_in_sla = self.include_in_sla + quality_dimension = self.quality_dimension + display_name = self.display_name + data_grouping = self.data_grouping + always_collect_error_samples = self.always_collect_error_samples + parameters: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.parameters, Unset): + parameters = self.parameters.to_dict() + + warning: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.warning, Unset): + warning = self.warning.to_dict() + + error: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.error, Unset): + error = self.error.to_dict() + + fatal: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.fatal, Unset): + fatal = self.fatal.to_dict() + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if schedule_override is not UNSET: + field_dict["schedule_override"] = schedule_override + if comments is not UNSET: + field_dict["comments"] = comments + if disabled is not UNSET: + field_dict["disabled"] = disabled + if exclude_from_kpi is not UNSET: + field_dict["exclude_from_kpi"] = exclude_from_kpi + if include_in_sla is not UNSET: + field_dict["include_in_sla"] = include_in_sla + if quality_dimension is not UNSET: + field_dict["quality_dimension"] = quality_dimension + if display_name is not UNSET: + field_dict["display_name"] = display_name + if data_grouping is not UNSET: + field_dict["data_grouping"] = data_grouping + if always_collect_error_samples is not UNSET: + field_dict["always_collect_error_samples"] = always_collect_error_samples + if parameters is not UNSET: + field_dict["parameters"] = parameters + if warning is not UNSET: + field_dict["warning"] = warning + if error is not UNSET: + field_dict["error"] = error + if fatal is not UNSET: + field_dict["fatal"] = fatal + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.comment_spec import CommentSpec + from ..models.max_count_rule_0_error_parameters_spec import ( + MaxCountRule0ErrorParametersSpec, + ) + from ..models.max_count_rule_0_warning_parameters_spec import ( + MaxCountRule0WarningParametersSpec, + ) + from ..models.max_count_rule_100_parameters_spec import ( + MaxCountRule100ParametersSpec, + ) + from ..models.monitoring_schedule_spec import MonitoringScheduleSpec + from ..models.table_duplicate_record_count_sensor_parameters_spec import ( + TableDuplicateRecordCountSensorParametersSpec, + ) + + d = src_dict.copy() + _schedule_override = d.pop("schedule_override", UNSET) + schedule_override: Union[Unset, MonitoringScheduleSpec] + if isinstance(_schedule_override, Unset): + schedule_override = UNSET + else: + schedule_override = MonitoringScheduleSpec.from_dict(_schedule_override) + + comments = [] + _comments = d.pop("comments", UNSET) + for comments_item_data in _comments or []: + comments_item = CommentSpec.from_dict(comments_item_data) + + comments.append(comments_item) + + disabled = d.pop("disabled", UNSET) + + exclude_from_kpi = d.pop("exclude_from_kpi", UNSET) + + include_in_sla = d.pop("include_in_sla", UNSET) + + quality_dimension = d.pop("quality_dimension", UNSET) + + display_name = d.pop("display_name", UNSET) + + data_grouping = d.pop("data_grouping", UNSET) + + always_collect_error_samples = d.pop("always_collect_error_samples", UNSET) + + _parameters = d.pop("parameters", UNSET) + parameters: Union[Unset, TableDuplicateRecordCountSensorParametersSpec] + if isinstance(_parameters, Unset): + parameters = UNSET + else: + parameters = TableDuplicateRecordCountSensorParametersSpec.from_dict( + _parameters + ) + + _warning = d.pop("warning", UNSET) + warning: Union[Unset, MaxCountRule0WarningParametersSpec] + if isinstance(_warning, Unset): + warning = UNSET + else: + warning = MaxCountRule0WarningParametersSpec.from_dict(_warning) + + _error = d.pop("error", UNSET) + error: Union[Unset, MaxCountRule0ErrorParametersSpec] + if isinstance(_error, Unset): + error = UNSET + else: + error = MaxCountRule0ErrorParametersSpec.from_dict(_error) + + _fatal = d.pop("fatal", UNSET) + fatal: Union[Unset, MaxCountRule100ParametersSpec] + if isinstance(_fatal, Unset): + fatal = UNSET + else: + fatal = MaxCountRule100ParametersSpec.from_dict(_fatal) + + table_duplicate_record_count_check_spec = cls( + schedule_override=schedule_override, + comments=comments, + disabled=disabled, + exclude_from_kpi=exclude_from_kpi, + include_in_sla=include_in_sla, + quality_dimension=quality_dimension, + display_name=display_name, + data_grouping=data_grouping, + always_collect_error_samples=always_collect_error_samples, + parameters=parameters, + warning=warning, + error=error, + fatal=fatal, + ) + + table_duplicate_record_count_check_spec.additional_properties = d + return table_duplicate_record_count_check_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_duplicate_record_count_sensor_parameters_spec.py b/distribution/python/dqops/client/models/table_duplicate_record_count_sensor_parameters_spec.py new file mode 100644 index 0000000000..291ec3a823 --- /dev/null +++ b/distribution/python/dqops/client/models/table_duplicate_record_count_sensor_parameters_spec.py @@ -0,0 +1,69 @@ +from typing import Any, Dict, List, Type, TypeVar, Union, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="TableDuplicateRecordCountSensorParametersSpec") + + +@_attrs_define +class TableDuplicateRecordCountSensorParametersSpec: + """ + Attributes: + filter_ (Union[Unset, str]): SQL WHERE clause added to the sensor query. Both the table level filter and a + sensor query filter are added, separated by an AND operator. + columns (Union[Unset, List[str]]): A list of columns used for uniqueness record duplicate verification. + """ + + filter_: Union[Unset, str] = UNSET + columns: Union[Unset, List[str]] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + filter_ = self.filter_ + columns: Union[Unset, List[str]] = UNSET + if not isinstance(self.columns, Unset): + columns = self.columns + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if filter_ is not UNSET: + field_dict["filter"] = filter_ + if columns is not UNSET: + field_dict["columns"] = columns + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + filter_ = d.pop("filter", UNSET) + + columns = cast(List[str], d.pop("columns", UNSET)) + + table_duplicate_record_count_sensor_parameters_spec = cls( + filter_=filter_, + columns=columns, + ) + + table_duplicate_record_count_sensor_parameters_spec.additional_properties = d + return table_duplicate_record_count_sensor_parameters_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_duplicate_record_percent_check_spec.py b/distribution/python/dqops/client/models/table_duplicate_record_percent_check_spec.py new file mode 100644 index 0000000000..8e6e459de0 --- /dev/null +++ b/distribution/python/dqops/client/models/table_duplicate_record_percent_check_spec.py @@ -0,0 +1,254 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.comment_spec import CommentSpec + from ..models.max_percent_rule_0_error_parameters_spec import ( + MaxPercentRule0ErrorParametersSpec, + ) + from ..models.max_percent_rule_0_warning_parameters_spec import ( + MaxPercentRule0WarningParametersSpec, + ) + from ..models.max_percent_rule_5_parameters_spec import ( + MaxPercentRule5ParametersSpec, + ) + from ..models.monitoring_schedule_spec import MonitoringScheduleSpec + from ..models.table_duplicate_record_percent_sensor_parameters_spec import ( + TableDuplicateRecordPercentSensorParametersSpec, + ) + + +T = TypeVar("T", bound="TableDuplicateRecordPercentCheckSpec") + + +@_attrs_define +class TableDuplicateRecordPercentCheckSpec: + """ + Attributes: + schedule_override (Union[Unset, MonitoringScheduleSpec]): + comments (Union[Unset, List['CommentSpec']]): Comments for change tracking. Please put comments in this + collection because YAML comments may be removed when the YAML file is modified by the tool (serialization and + deserialization will remove non tracked comments). + disabled (Union[Unset, bool]): Disables the data quality check. Only enabled data quality checks and monitorings + are executed. The check should be disabled if it should not work, but the configuration of the sensor and rules + should be preserved in the configuration. + exclude_from_kpi (Union[Unset, bool]): Data quality check results (alerts) are included in the data quality KPI + calculation by default. Set this field to true in order to exclude this data quality check from the data quality + KPI calculation. + include_in_sla (Union[Unset, bool]): Marks the data quality check as part of a data quality SLA (Data Contract). + The data quality SLA is a set of critical data quality checks that must always pass and are considered as a Data + Contract for the dataset. + quality_dimension (Union[Unset, str]): Configures a custom data quality dimension name that is different than + the built-in dimensions (Timeliness, Validity, etc.). + display_name (Union[Unset, str]): Data quality check display name that can be assigned to the check, otherwise + the check_display_name stored in the parquet result files is the check_name. + data_grouping (Union[Unset, str]): Data grouping configuration name that should be applied to this data quality + check. The data grouping is used to group the check's result by a GROUP BY clause in SQL, evaluating the data + quality check for each group of rows. Use the name of one of data grouping configurations defined on the parent + table. + always_collect_error_samples (Union[Unset, bool]): Forces collecting error samples for this check whenever it + fails, even if it is a monitoring check that is run by a scheduler, and running an additional query to collect + error samples will impose additional load on the data source. + parameters (Union[Unset, TableDuplicateRecordPercentSensorParametersSpec]): + warning (Union[Unset, MaxPercentRule0WarningParametersSpec]): + error (Union[Unset, MaxPercentRule0ErrorParametersSpec]): + fatal (Union[Unset, MaxPercentRule5ParametersSpec]): + """ + + schedule_override: Union[Unset, "MonitoringScheduleSpec"] = UNSET + comments: Union[Unset, List["CommentSpec"]] = UNSET + disabled: Union[Unset, bool] = UNSET + exclude_from_kpi: Union[Unset, bool] = UNSET + include_in_sla: Union[Unset, bool] = UNSET + quality_dimension: Union[Unset, str] = UNSET + display_name: Union[Unset, str] = UNSET + data_grouping: Union[Unset, str] = UNSET + always_collect_error_samples: Union[Unset, bool] = UNSET + parameters: Union[Unset, "TableDuplicateRecordPercentSensorParametersSpec"] = UNSET + warning: Union[Unset, "MaxPercentRule0WarningParametersSpec"] = UNSET + error: Union[Unset, "MaxPercentRule0ErrorParametersSpec"] = UNSET + fatal: Union[Unset, "MaxPercentRule5ParametersSpec"] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + schedule_override: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.schedule_override, Unset): + schedule_override = self.schedule_override.to_dict() + + comments: Union[Unset, List[Dict[str, Any]]] = UNSET + if not isinstance(self.comments, Unset): + comments = [] + for comments_item_data in self.comments: + comments_item = comments_item_data.to_dict() + + comments.append(comments_item) + + disabled = self.disabled + exclude_from_kpi = self.exclude_from_kpi + include_in_sla = self.include_in_sla + quality_dimension = self.quality_dimension + display_name = self.display_name + data_grouping = self.data_grouping + always_collect_error_samples = self.always_collect_error_samples + parameters: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.parameters, Unset): + parameters = self.parameters.to_dict() + + warning: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.warning, Unset): + warning = self.warning.to_dict() + + error: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.error, Unset): + error = self.error.to_dict() + + fatal: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.fatal, Unset): + fatal = self.fatal.to_dict() + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if schedule_override is not UNSET: + field_dict["schedule_override"] = schedule_override + if comments is not UNSET: + field_dict["comments"] = comments + if disabled is not UNSET: + field_dict["disabled"] = disabled + if exclude_from_kpi is not UNSET: + field_dict["exclude_from_kpi"] = exclude_from_kpi + if include_in_sla is not UNSET: + field_dict["include_in_sla"] = include_in_sla + if quality_dimension is not UNSET: + field_dict["quality_dimension"] = quality_dimension + if display_name is not UNSET: + field_dict["display_name"] = display_name + if data_grouping is not UNSET: + field_dict["data_grouping"] = data_grouping + if always_collect_error_samples is not UNSET: + field_dict["always_collect_error_samples"] = always_collect_error_samples + if parameters is not UNSET: + field_dict["parameters"] = parameters + if warning is not UNSET: + field_dict["warning"] = warning + if error is not UNSET: + field_dict["error"] = error + if fatal is not UNSET: + field_dict["fatal"] = fatal + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.comment_spec import CommentSpec + from ..models.max_percent_rule_0_error_parameters_spec import ( + MaxPercentRule0ErrorParametersSpec, + ) + from ..models.max_percent_rule_0_warning_parameters_spec import ( + MaxPercentRule0WarningParametersSpec, + ) + from ..models.max_percent_rule_5_parameters_spec import ( + MaxPercentRule5ParametersSpec, + ) + from ..models.monitoring_schedule_spec import MonitoringScheduleSpec + from ..models.table_duplicate_record_percent_sensor_parameters_spec import ( + TableDuplicateRecordPercentSensorParametersSpec, + ) + + d = src_dict.copy() + _schedule_override = d.pop("schedule_override", UNSET) + schedule_override: Union[Unset, MonitoringScheduleSpec] + if isinstance(_schedule_override, Unset): + schedule_override = UNSET + else: + schedule_override = MonitoringScheduleSpec.from_dict(_schedule_override) + + comments = [] + _comments = d.pop("comments", UNSET) + for comments_item_data in _comments or []: + comments_item = CommentSpec.from_dict(comments_item_data) + + comments.append(comments_item) + + disabled = d.pop("disabled", UNSET) + + exclude_from_kpi = d.pop("exclude_from_kpi", UNSET) + + include_in_sla = d.pop("include_in_sla", UNSET) + + quality_dimension = d.pop("quality_dimension", UNSET) + + display_name = d.pop("display_name", UNSET) + + data_grouping = d.pop("data_grouping", UNSET) + + always_collect_error_samples = d.pop("always_collect_error_samples", UNSET) + + _parameters = d.pop("parameters", UNSET) + parameters: Union[Unset, TableDuplicateRecordPercentSensorParametersSpec] + if isinstance(_parameters, Unset): + parameters = UNSET + else: + parameters = TableDuplicateRecordPercentSensorParametersSpec.from_dict( + _parameters + ) + + _warning = d.pop("warning", UNSET) + warning: Union[Unset, MaxPercentRule0WarningParametersSpec] + if isinstance(_warning, Unset): + warning = UNSET + else: + warning = MaxPercentRule0WarningParametersSpec.from_dict(_warning) + + _error = d.pop("error", UNSET) + error: Union[Unset, MaxPercentRule0ErrorParametersSpec] + if isinstance(_error, Unset): + error = UNSET + else: + error = MaxPercentRule0ErrorParametersSpec.from_dict(_error) + + _fatal = d.pop("fatal", UNSET) + fatal: Union[Unset, MaxPercentRule5ParametersSpec] + if isinstance(_fatal, Unset): + fatal = UNSET + else: + fatal = MaxPercentRule5ParametersSpec.from_dict(_fatal) + + table_duplicate_record_percent_check_spec = cls( + schedule_override=schedule_override, + comments=comments, + disabled=disabled, + exclude_from_kpi=exclude_from_kpi, + include_in_sla=include_in_sla, + quality_dimension=quality_dimension, + display_name=display_name, + data_grouping=data_grouping, + always_collect_error_samples=always_collect_error_samples, + parameters=parameters, + warning=warning, + error=error, + fatal=fatal, + ) + + table_duplicate_record_percent_check_spec.additional_properties = d + return table_duplicate_record_percent_check_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_duplicate_record_percent_sensor_parameters_spec.py b/distribution/python/dqops/client/models/table_duplicate_record_percent_sensor_parameters_spec.py new file mode 100644 index 0000000000..3315adbded --- /dev/null +++ b/distribution/python/dqops/client/models/table_duplicate_record_percent_sensor_parameters_spec.py @@ -0,0 +1,69 @@ +from typing import Any, Dict, List, Type, TypeVar, Union, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="TableDuplicateRecordPercentSensorParametersSpec") + + +@_attrs_define +class TableDuplicateRecordPercentSensorParametersSpec: + """ + Attributes: + filter_ (Union[Unset, str]): SQL WHERE clause added to the sensor query. Both the table level filter and a + sensor query filter are added, separated by an AND operator. + columns (Union[Unset, List[str]]): A list of columns used for uniqueness record duplicate verification. + """ + + filter_: Union[Unset, str] = UNSET + columns: Union[Unset, List[str]] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + filter_ = self.filter_ + columns: Union[Unset, List[str]] = UNSET + if not isinstance(self.columns, Unset): + columns = self.columns + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if filter_ is not UNSET: + field_dict["filter"] = filter_ + if columns is not UNSET: + field_dict["columns"] = columns + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + filter_ = d.pop("filter", UNSET) + + columns = cast(List[str], d.pop("columns", UNSET)) + + table_duplicate_record_percent_sensor_parameters_spec = cls( + filter_=filter_, + columns=columns, + ) + + table_duplicate_record_percent_sensor_parameters_spec.additional_properties = d + return table_duplicate_record_percent_sensor_parameters_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_lineage_source_list_model.py b/distribution/python/dqops/client/models/table_lineage_source_list_model.py new file mode 100644 index 0000000000..f48bd7bf3e --- /dev/null +++ b/distribution/python/dqops/client/models/table_lineage_source_list_model.py @@ -0,0 +1,187 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.table_current_data_quality_status_model import ( + TableCurrentDataQualityStatusModel, + ) + from ..models.table_lineage_source_list_model_properties import ( + TableLineageSourceListModelProperties, + ) + + +T = TypeVar("T", bound="TableLineageSourceListModel") + + +@_attrs_define +class TableLineageSourceListModel: + """Data lineage model that describes one source table of the current table. + + Attributes: + target_connection (Union[Unset, str]): The connection name where the target table is defined. + target_schema (Union[Unset, str]): The schema name in the target connection where the target table is defined. + target_table (Union[Unset, str]): The name of the target table inside the target schema. + source_connection (Union[Unset, str]): The name of a source connection that is defined in DQOps and contains a + source table from which the current table receives data. + source_schema (Union[Unset, str]): The name of a source schema within the source connection that contains a + source table from which the current table receives data. + source_table (Union[Unset, str]): The name of a source schema within the source connection that contains a + source table from which the current table receives data. + data_lineage_source_tool (Union[Unset, str]): The name of a source tool from which this data lineage information + was copied. This field should be filled when the data lineage was imported from another data catalog or a data + lineage tracking platform. + properties (Union[Unset, TableLineageSourceListModelProperties]): A dictionary of mapping properties stored as a + key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external + data lineage sources can use it to store mapping information. + can_edit (Union[Unset, bool]): Boolean flag that decides if the current user can update or delete this object. + source_table_data_quality_status (Union[Unset, TableCurrentDataQualityStatusModel]): The table's most recent + data quality status. It is a summary of the results of the most recently executed data quality checks on the + table. Verify the value of the highest_severity_level to see if there are any data quality issues on the table. + The values of severity levels are: 0 - all data quality checks passed, 1 - a warning was detected, 2 - an error + was detected, 3 - a fatal data quality issue was detected. + """ + + target_connection: Union[Unset, str] = UNSET + target_schema: Union[Unset, str] = UNSET + target_table: Union[Unset, str] = UNSET + source_connection: Union[Unset, str] = UNSET + source_schema: Union[Unset, str] = UNSET + source_table: Union[Unset, str] = UNSET + data_lineage_source_tool: Union[Unset, str] = UNSET + properties: Union[Unset, "TableLineageSourceListModelProperties"] = UNSET + can_edit: Union[Unset, bool] = UNSET + source_table_data_quality_status: Union[ + Unset, "TableCurrentDataQualityStatusModel" + ] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + target_connection = self.target_connection + target_schema = self.target_schema + target_table = self.target_table + source_connection = self.source_connection + source_schema = self.source_schema + source_table = self.source_table + data_lineage_source_tool = self.data_lineage_source_tool + properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.properties, Unset): + properties = self.properties.to_dict() + + can_edit = self.can_edit + source_table_data_quality_status: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.source_table_data_quality_status, Unset): + source_table_data_quality_status = ( + self.source_table_data_quality_status.to_dict() + ) + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if target_connection is not UNSET: + field_dict["target_connection"] = target_connection + if target_schema is not UNSET: + field_dict["target_schema"] = target_schema + if target_table is not UNSET: + field_dict["target_table"] = target_table + if source_connection is not UNSET: + field_dict["source_connection"] = source_connection + if source_schema is not UNSET: + field_dict["source_schema"] = source_schema + if source_table is not UNSET: + field_dict["source_table"] = source_table + if data_lineage_source_tool is not UNSET: + field_dict["data_lineage_source_tool"] = data_lineage_source_tool + if properties is not UNSET: + field_dict["properties"] = properties + if can_edit is not UNSET: + field_dict["can_edit"] = can_edit + if source_table_data_quality_status is not UNSET: + field_dict["source_table_data_quality_status"] = ( + source_table_data_quality_status + ) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.table_current_data_quality_status_model import ( + TableCurrentDataQualityStatusModel, + ) + from ..models.table_lineage_source_list_model_properties import ( + TableLineageSourceListModelProperties, + ) + + d = src_dict.copy() + target_connection = d.pop("target_connection", UNSET) + + target_schema = d.pop("target_schema", UNSET) + + target_table = d.pop("target_table", UNSET) + + source_connection = d.pop("source_connection", UNSET) + + source_schema = d.pop("source_schema", UNSET) + + source_table = d.pop("source_table", UNSET) + + data_lineage_source_tool = d.pop("data_lineage_source_tool", UNSET) + + _properties = d.pop("properties", UNSET) + properties: Union[Unset, TableLineageSourceListModelProperties] + if isinstance(_properties, Unset): + properties = UNSET + else: + properties = TableLineageSourceListModelProperties.from_dict(_properties) + + can_edit = d.pop("can_edit", UNSET) + + _source_table_data_quality_status = d.pop( + "source_table_data_quality_status", UNSET + ) + source_table_data_quality_status: Union[ + Unset, TableCurrentDataQualityStatusModel + ] + if isinstance(_source_table_data_quality_status, Unset): + source_table_data_quality_status = UNSET + else: + source_table_data_quality_status = ( + TableCurrentDataQualityStatusModel.from_dict( + _source_table_data_quality_status + ) + ) + + table_lineage_source_list_model = cls( + target_connection=target_connection, + target_schema=target_schema, + target_table=target_table, + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + data_lineage_source_tool=data_lineage_source_tool, + properties=properties, + can_edit=can_edit, + source_table_data_quality_status=source_table_data_quality_status, + ) + + table_lineage_source_list_model.additional_properties = d + return table_lineage_source_list_model + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_lineage_source_list_model_properties.py b/distribution/python/dqops/client/models/table_lineage_source_list_model_properties.py new file mode 100644 index 0000000000..4161871262 --- /dev/null +++ b/distribution/python/dqops/client/models/table_lineage_source_list_model_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="TableLineageSourceListModelProperties") + + +@_attrs_define +class TableLineageSourceListModelProperties: + """A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are + importing data lineage mappings from external data lineage sources can use it to store mapping information. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + table_lineage_source_list_model_properties = cls() + + table_lineage_source_list_model_properties.additional_properties = d + return table_lineage_source_list_model_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_lineage_source_spec.py b/distribution/python/dqops/client/models/table_lineage_source_spec.py new file mode 100644 index 0000000000..eba576dfaa --- /dev/null +++ b/distribution/python/dqops/client/models/table_lineage_source_spec.py @@ -0,0 +1,136 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.table_lineage_source_spec_columns import TableLineageSourceSpecColumns + from ..models.table_lineage_source_spec_properties import ( + TableLineageSourceSpecProperties, + ) + + +T = TypeVar("T", bound="TableLineageSourceSpec") + + +@_attrs_define +class TableLineageSourceSpec: + """ + Attributes: + source_connection (Union[Unset, str]): The name of a source connection that is defined in DQOps and contains a + source table from which the current table receives data. + source_schema (Union[Unset, str]): The name of a source schema within the source connection that contains a + source table from which the current table receives data. + source_table (Union[Unset, str]): The name of a source table in the source schema from which the current table + receives data. + data_lineage_source_tool (Union[Unset, str]): The name of a source tool from which this data lineage information + was copied. This field should be filled when the data lineage was imported from another data catalog or a data + lineage tracking platform. + properties (Union[Unset, TableLineageSourceSpecProperties]): A dictionary of mapping properties stored as a + key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external + data lineage sources can use it to store mapping information. + columns (Union[Unset, TableLineageSourceSpecColumns]): Configuration of source columns for each column in the + current table. The keys in this dictionary are column names in the current table. The object stored in the + dictionary contain a list of source columns. + """ + + source_connection: Union[Unset, str] = UNSET + source_schema: Union[Unset, str] = UNSET + source_table: Union[Unset, str] = UNSET + data_lineage_source_tool: Union[Unset, str] = UNSET + properties: Union[Unset, "TableLineageSourceSpecProperties"] = UNSET + columns: Union[Unset, "TableLineageSourceSpecColumns"] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + source_connection = self.source_connection + source_schema = self.source_schema + source_table = self.source_table + data_lineage_source_tool = self.data_lineage_source_tool + properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.properties, Unset): + properties = self.properties.to_dict() + + columns: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.columns, Unset): + columns = self.columns.to_dict() + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if source_connection is not UNSET: + field_dict["source_connection"] = source_connection + if source_schema is not UNSET: + field_dict["source_schema"] = source_schema + if source_table is not UNSET: + field_dict["source_table"] = source_table + if data_lineage_source_tool is not UNSET: + field_dict["data_lineage_source_tool"] = data_lineage_source_tool + if properties is not UNSET: + field_dict["properties"] = properties + if columns is not UNSET: + field_dict["columns"] = columns + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.table_lineage_source_spec_columns import ( + TableLineageSourceSpecColumns, + ) + from ..models.table_lineage_source_spec_properties import ( + TableLineageSourceSpecProperties, + ) + + d = src_dict.copy() + source_connection = d.pop("source_connection", UNSET) + + source_schema = d.pop("source_schema", UNSET) + + source_table = d.pop("source_table", UNSET) + + data_lineage_source_tool = d.pop("data_lineage_source_tool", UNSET) + + _properties = d.pop("properties", UNSET) + properties: Union[Unset, TableLineageSourceSpecProperties] + if isinstance(_properties, Unset): + properties = UNSET + else: + properties = TableLineageSourceSpecProperties.from_dict(_properties) + + _columns = d.pop("columns", UNSET) + columns: Union[Unset, TableLineageSourceSpecColumns] + if isinstance(_columns, Unset): + columns = UNSET + else: + columns = TableLineageSourceSpecColumns.from_dict(_columns) + + table_lineage_source_spec = cls( + source_connection=source_connection, + source_schema=source_schema, + source_table=source_table, + data_lineage_source_tool=data_lineage_source_tool, + properties=properties, + columns=columns, + ) + + table_lineage_source_spec.additional_properties = d + return table_lineage_source_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_lineage_source_spec_columns.py b/distribution/python/dqops/client/models/table_lineage_source_spec_columns.py new file mode 100644 index 0000000000..282c840989 --- /dev/null +++ b/distribution/python/dqops/client/models/table_lineage_source_spec_columns.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.column_lineage_source_spec import ColumnLineageSourceSpec + + +T = TypeVar("T", bound="TableLineageSourceSpecColumns") + + +@_attrs_define +class TableLineageSourceSpecColumns: + """Configuration of source columns for each column in the current table. The keys in this dictionary are column names + in the current table. The object stored in the dictionary contain a list of source columns. + + """ + + additional_properties: Dict[str, "ColumnLineageSourceSpec"] = _attrs_field( + init=False, factory=dict + ) + + def to_dict(self) -> Dict[str, Any]: + pass + + field_dict: Dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.column_lineage_source_spec import ColumnLineageSourceSpec + + d = src_dict.copy() + table_lineage_source_spec_columns = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = ColumnLineageSourceSpec.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + table_lineage_source_spec_columns.additional_properties = additional_properties + return table_lineage_source_spec_columns + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> "ColumnLineageSourceSpec": + return self.additional_properties[key] + + def __setitem__(self, key: str, value: "ColumnLineageSourceSpec") -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_lineage_source_spec_properties.py b/distribution/python/dqops/client/models/table_lineage_source_spec_properties.py new file mode 100644 index 0000000000..6d76c1256c --- /dev/null +++ b/distribution/python/dqops/client/models/table_lineage_source_spec_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="TableLineageSourceSpecProperties") + + +@_attrs_define +class TableLineageSourceSpecProperties: + """A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are + importing data lineage mappings from external data lineage sources can use it to store mapping information. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + table_lineage_source_spec_properties = cls() + + table_lineage_source_spec_properties.additional_properties = d + return table_lineage_source_spec_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_list_model.py b/distribution/python/dqops/client/models/table_list_model.py index 98e6d23f2b..af7118e73b 100644 --- a/distribution/python/dqops/client/models/table_list_model.py +++ b/distribution/python/dqops/client/models/table_list_model.py @@ -19,6 +19,9 @@ from ..models.table_current_data_quality_status_model import ( TableCurrentDataQualityStatusModel, ) + from ..models.table_list_model_advanced_properties import ( + TableListModelAdvancedProperties, + ) from ..models.table_owner_spec import TableOwnerSpec @@ -76,6 +79,8 @@ class TableListModel: identifies which checks on which tables and columns should be executed. collect_statistics_job_template (Union[Unset, StatisticsCollectorSearchFilters]): data_clean_job_template (Union[Unset, DeleteStoredDataQueueJobParameters]): + advanced_properties (Union[Unset, TableListModelAdvancedProperties]): A dictionary of advanced properties that + can be used for e.g. to support mapping data to data catalogs, a key/value dictionary. can_edit (Union[Unset, bool]): Boolean flag that decides if the current user can update or delete this object. can_collect_statistics (Union[Unset, bool]): Boolean flag that decides if the current user can collect statistics. @@ -115,6 +120,7 @@ class TableListModel: Unset, "StatisticsCollectorSearchFilters" ] = UNSET data_clean_job_template: Union[Unset, "DeleteStoredDataQueueJobParameters"] = UNSET + advanced_properties: Union[Unset, "TableListModelAdvancedProperties"] = UNSET can_edit: Union[Unset, bool] = UNSET can_collect_statistics: Union[Unset, bool] = UNSET can_run_checks: Union[Unset, bool] = UNSET @@ -198,6 +204,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.data_clean_job_template, Unset): data_clean_job_template = self.data_clean_job_template.to_dict() + advanced_properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.advanced_properties, Unset): + advanced_properties = self.advanced_properties.to_dict() + can_edit = self.can_edit can_collect_statistics = self.can_collect_statistics can_run_checks = self.can_run_checks @@ -279,6 +289,8 @@ def to_dict(self) -> Dict[str, Any]: ) if data_clean_job_template is not UNSET: field_dict["data_clean_job_template"] = data_clean_job_template + if advanced_properties is not UNSET: + field_dict["advanced_properties"] = advanced_properties if can_edit is not UNSET: field_dict["can_edit"] = can_edit if can_collect_statistics is not UNSET: @@ -306,6 +318,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.table_current_data_quality_status_model import ( TableCurrentDataQualityStatusModel, ) + from ..models.table_list_model_advanced_properties import ( + TableListModelAdvancedProperties, + ) from ..models.table_owner_spec import TableOwnerSpec d = src_dict.copy() @@ -454,6 +469,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: _data_clean_job_template ) + _advanced_properties = d.pop("advanced_properties", UNSET) + advanced_properties: Union[Unset, TableListModelAdvancedProperties] + if isinstance(_advanced_properties, Unset): + advanced_properties = UNSET + else: + advanced_properties = TableListModelAdvancedProperties.from_dict( + _advanced_properties + ) + can_edit = d.pop("can_edit", UNSET) can_collect_statistics = d.pop("can_collect_statistics", UNSET) @@ -490,6 +514,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: run_partition_checks_job_template=run_partition_checks_job_template, collect_statistics_job_template=collect_statistics_job_template, data_clean_job_template=data_clean_job_template, + advanced_properties=advanced_properties, can_edit=can_edit, can_collect_statistics=can_collect_statistics, can_run_checks=can_run_checks, diff --git a/distribution/python/dqops/client/models/table_list_model_advanced_properties.py b/distribution/python/dqops/client/models/table_list_model_advanced_properties.py new file mode 100644 index 0000000000..4107315d70 --- /dev/null +++ b/distribution/python/dqops/client/models/table_list_model_advanced_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="TableListModelAdvancedProperties") + + +@_attrs_define +class TableListModelAdvancedProperties: + """A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value + dictionary. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + table_list_model_advanced_properties = cls() + + table_list_model_advanced_properties.additional_properties = d + return table_list_model_advanced_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_monthly_monitoring_check_categories_spec.py b/distribution/python/dqops/client/models/table_monthly_monitoring_check_categories_spec.py index a4baf729e0..df6dd0cc92 100644 --- a/distribution/python/dqops/client/models/table_monthly_monitoring_check_categories_spec.py +++ b/distribution/python/dqops/client/models/table_monthly_monitoring_check_categories_spec.py @@ -27,6 +27,9 @@ from ..models.table_timeliness_monthly_monitoring_checks_spec import ( TableTimelinessMonthlyMonitoringChecksSpec, ) + from ..models.table_uniqueness_monthly_monitoring_checks_spec import ( + TableUniquenessMonthlyMonitoringChecksSpec, + ) from ..models.table_volume_monthly_monitoring_checks_spec import ( TableVolumeMonthlyMonitoringChecksSpec, ) @@ -47,6 +50,7 @@ class TableMonthlyMonitoringCheckCategoriesSpec: custom_sql (Union[Unset, TableCustomSqlMonthlyMonitoringChecksSpec]): availability (Union[Unset, TableAvailabilityMonthlyMonitoringChecksSpec]): schema (Union[Unset, TableSchemaMonthlyMonitoringChecksSpec]): + uniqueness (Union[Unset, TableUniquenessMonthlyMonitoringChecksSpec]): comparisons (Union[Unset, TableMonthlyMonitoringCheckCategoriesSpecComparisons]): Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table. @@ -59,6 +63,7 @@ class TableMonthlyMonitoringCheckCategoriesSpec: custom_sql: Union[Unset, "TableCustomSqlMonthlyMonitoringChecksSpec"] = UNSET availability: Union[Unset, "TableAvailabilityMonthlyMonitoringChecksSpec"] = UNSET schema: Union[Unset, "TableSchemaMonthlyMonitoringChecksSpec"] = UNSET + uniqueness: Union[Unset, "TableUniquenessMonthlyMonitoringChecksSpec"] = UNSET comparisons: Union[ Unset, "TableMonthlyMonitoringCheckCategoriesSpecComparisons" ] = UNSET @@ -93,6 +98,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.schema, Unset): schema = self.schema.to_dict() + uniqueness: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.uniqueness, Unset): + uniqueness = self.uniqueness.to_dict() + comparisons: Union[Unset, Dict[str, Any]] = UNSET if not isinstance(self.comparisons, Unset): comparisons = self.comparisons.to_dict() @@ -114,6 +123,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["availability"] = availability if schema is not UNSET: field_dict["schema"] = schema + if uniqueness is not UNSET: + field_dict["uniqueness"] = uniqueness if comparisons is not UNSET: field_dict["comparisons"] = comparisons @@ -142,6 +153,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.table_timeliness_monthly_monitoring_checks_spec import ( TableTimelinessMonthlyMonitoringChecksSpec, ) + from ..models.table_uniqueness_monthly_monitoring_checks_spec import ( + TableUniquenessMonthlyMonitoringChecksSpec, + ) from ..models.table_volume_monthly_monitoring_checks_spec import ( TableVolumeMonthlyMonitoringChecksSpec, ) @@ -202,6 +216,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: schema = TableSchemaMonthlyMonitoringChecksSpec.from_dict(_schema) + _uniqueness = d.pop("uniqueness", UNSET) + uniqueness: Union[Unset, TableUniquenessMonthlyMonitoringChecksSpec] + if isinstance(_uniqueness, Unset): + uniqueness = UNSET + else: + uniqueness = TableUniquenessMonthlyMonitoringChecksSpec.from_dict( + _uniqueness + ) + _comparisons = d.pop("comparisons", UNSET) comparisons: Union[Unset, TableMonthlyMonitoringCheckCategoriesSpecComparisons] if isinstance(_comparisons, Unset): @@ -221,6 +244,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: custom_sql=custom_sql, availability=availability, schema=schema, + uniqueness=uniqueness, comparisons=comparisons, ) diff --git a/distribution/python/dqops/client/models/table_monthly_partitioned_check_categories_spec.py b/distribution/python/dqops/client/models/table_monthly_partitioned_check_categories_spec.py index a182dc8b99..b5264825c1 100644 --- a/distribution/python/dqops/client/models/table_monthly_partitioned_check_categories_spec.py +++ b/distribution/python/dqops/client/models/table_monthly_partitioned_check_categories_spec.py @@ -18,6 +18,9 @@ from ..models.table_timeliness_monthly_partitioned_checks_spec import ( TableTimelinessMonthlyPartitionedChecksSpec, ) + from ..models.table_uniqueness_monthly_partition_checks_spec import ( + TableUniquenessMonthlyPartitionChecksSpec, + ) from ..models.table_volume_monthly_partitioned_checks_spec import ( TableVolumeMonthlyPartitionedChecksSpec, ) @@ -35,6 +38,7 @@ class TableMonthlyPartitionedCheckCategoriesSpec: volume (Union[Unset, TableVolumeMonthlyPartitionedChecksSpec]): timeliness (Union[Unset, TableTimelinessMonthlyPartitionedChecksSpec]): custom_sql (Union[Unset, TableCustomSqlMonthlyPartitionedChecksSpec]): + uniqueness (Union[Unset, TableUniquenessMonthlyPartitionChecksSpec]): comparisons (Union[Unset, TableMonthlyPartitionedCheckCategoriesSpecComparisons]): Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table. @@ -44,6 +48,7 @@ class TableMonthlyPartitionedCheckCategoriesSpec: volume: Union[Unset, "TableVolumeMonthlyPartitionedChecksSpec"] = UNSET timeliness: Union[Unset, "TableTimelinessMonthlyPartitionedChecksSpec"] = UNSET custom_sql: Union[Unset, "TableCustomSqlMonthlyPartitionedChecksSpec"] = UNSET + uniqueness: Union[Unset, "TableUniquenessMonthlyPartitionChecksSpec"] = UNSET comparisons: Union[ Unset, "TableMonthlyPartitionedCheckCategoriesSpecComparisons" ] = UNSET @@ -66,6 +71,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.custom_sql, Unset): custom_sql = self.custom_sql.to_dict() + uniqueness: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.uniqueness, Unset): + uniqueness = self.uniqueness.to_dict() + comparisons: Union[Unset, Dict[str, Any]] = UNSET if not isinstance(self.comparisons, Unset): comparisons = self.comparisons.to_dict() @@ -81,6 +90,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["timeliness"] = timeliness if custom_sql is not UNSET: field_dict["custom_sql"] = custom_sql + if uniqueness is not UNSET: + field_dict["uniqueness"] = uniqueness if comparisons is not UNSET: field_dict["comparisons"] = comparisons @@ -100,6 +111,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.table_timeliness_monthly_partitioned_checks_spec import ( TableTimelinessMonthlyPartitionedChecksSpec, ) + from ..models.table_uniqueness_monthly_partition_checks_spec import ( + TableUniquenessMonthlyPartitionChecksSpec, + ) from ..models.table_volume_monthly_partitioned_checks_spec import ( TableVolumeMonthlyPartitionedChecksSpec, ) @@ -137,6 +151,15 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: _custom_sql ) + _uniqueness = d.pop("uniqueness", UNSET) + uniqueness: Union[Unset, TableUniquenessMonthlyPartitionChecksSpec] + if isinstance(_uniqueness, Unset): + uniqueness = UNSET + else: + uniqueness = TableUniquenessMonthlyPartitionChecksSpec.from_dict( + _uniqueness + ) + _comparisons = d.pop("comparisons", UNSET) comparisons: Union[Unset, TableMonthlyPartitionedCheckCategoriesSpecComparisons] if isinstance(_comparisons, Unset): @@ -153,6 +176,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: volume=volume, timeliness=timeliness, custom_sql=custom_sql, + uniqueness=uniqueness, comparisons=comparisons, ) diff --git a/distribution/python/dqops/client/models/table_profiling_check_categories_spec.py b/distribution/python/dqops/client/models/table_profiling_check_categories_spec.py index 0f7d61e1c1..1830a6a021 100644 --- a/distribution/python/dqops/client/models/table_profiling_check_categories_spec.py +++ b/distribution/python/dqops/client/models/table_profiling_check_categories_spec.py @@ -28,6 +28,9 @@ from ..models.table_timeliness_profiling_checks_spec import ( TableTimelinessProfilingChecksSpec, ) + from ..models.table_uniqueness_profiling_checks_spec import ( + TableUniquenessProfilingChecksSpec, + ) from ..models.table_volume_profiling_checks_spec import ( TableVolumeProfilingChecksSpec, ) @@ -49,6 +52,7 @@ class TableProfilingCheckCategoriesSpec: custom_sql (Union[Unset, TableCustomSqlProfilingChecksSpec]): availability (Union[Unset, TableAvailabilityProfilingChecksSpec]): schema (Union[Unset, TableSchemaProfilingChecksSpec]): + uniqueness (Union[Unset, TableUniquenessProfilingChecksSpec]): comparisons (Union[Unset, TableProfilingCheckCategoriesSpecComparisons]): Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table. @@ -62,6 +66,7 @@ class TableProfilingCheckCategoriesSpec: custom_sql: Union[Unset, "TableCustomSqlProfilingChecksSpec"] = UNSET availability: Union[Unset, "TableAvailabilityProfilingChecksSpec"] = UNSET schema: Union[Unset, "TableSchemaProfilingChecksSpec"] = UNSET + uniqueness: Union[Unset, "TableUniquenessProfilingChecksSpec"] = UNSET comparisons: Union[Unset, "TableProfilingCheckCategoriesSpecComparisons"] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) @@ -98,6 +103,10 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.schema, Unset): schema = self.schema.to_dict() + uniqueness: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.uniqueness, Unset): + uniqueness = self.uniqueness.to_dict() + comparisons: Union[Unset, Dict[str, Any]] = UNSET if not isinstance(self.comparisons, Unset): comparisons = self.comparisons.to_dict() @@ -121,6 +130,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["availability"] = availability if schema is not UNSET: field_dict["schema"] = schema + if uniqueness is not UNSET: + field_dict["uniqueness"] = uniqueness if comparisons is not UNSET: field_dict["comparisons"] = comparisons @@ -149,6 +160,9 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.table_timeliness_profiling_checks_spec import ( TableTimelinessProfilingChecksSpec, ) + from ..models.table_uniqueness_profiling_checks_spec import ( + TableUniquenessProfilingChecksSpec, + ) from ..models.table_volume_profiling_checks_spec import ( TableVolumeProfilingChecksSpec, ) @@ -210,6 +224,13 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: schema = TableSchemaProfilingChecksSpec.from_dict(_schema) + _uniqueness = d.pop("uniqueness", UNSET) + uniqueness: Union[Unset, TableUniquenessProfilingChecksSpec] + if isinstance(_uniqueness, Unset): + uniqueness = UNSET + else: + uniqueness = TableUniquenessProfilingChecksSpec.from_dict(_uniqueness) + _comparisons = d.pop("comparisons", UNSET) comparisons: Union[Unset, TableProfilingCheckCategoriesSpecComparisons] if isinstance(_comparisons, Unset): @@ -228,6 +249,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: custom_sql=custom_sql, availability=availability, schema=schema, + uniqueness=uniqueness, comparisons=comparisons, ) diff --git a/distribution/python/dqops/client/models/default_table_checks_pattern_list_model.py b/distribution/python/dqops/client/models/table_quality_policy_list_model.py similarity index 82% rename from distribution/python/dqops/client/models/default_table_checks_pattern_list_model.py rename to distribution/python/dqops/client/models/table_quality_policy_list_model.py index ff2cee78b2..36ff98f208 100644 --- a/distribution/python/dqops/client/models/default_table_checks_pattern_list_model.py +++ b/distribution/python/dqops/client/models/table_quality_policy_list_model.py @@ -9,17 +9,17 @@ from ..models.target_table_pattern_spec import TargetTablePatternSpec -T = TypeVar("T", bound="DefaultTableChecksPatternListModel") +T = TypeVar("T", bound="TableQualityPolicyListModel") @_attrs_define -class DefaultTableChecksPatternListModel: - """Default table-level checks pattern list model +class TableQualityPolicyListModel: + """Default table-level checks pattern (data quality policy) list model Attributes: - pattern_name (Union[Unset, str]): Pattern name. - priority (Union[Unset, int]): The priority of the pattern. Patterns with lower values are applied before - patterns with higher priority values. + policy_name (Union[Unset, str]): Quality policy name. + priority (Union[Unset, int]): The priority of the policy. Policies with lower values are applied before policies + with higher priority values. disabled (Union[Unset, bool]): Disables this data quality check configuration. The checks will not be activated. description (Union[Unset, str]): The description (documentation) of this data quality check configuration. target_table (Union[Unset, TargetTablePatternSpec]): @@ -29,7 +29,7 @@ class DefaultTableChecksPatternListModel: error message and the file location. """ - pattern_name: Union[Unset, str] = UNSET + policy_name: Union[Unset, str] = UNSET priority: Union[Unset, int] = UNSET disabled: Union[Unset, bool] = UNSET description: Union[Unset, str] = UNSET @@ -39,7 +39,7 @@ class DefaultTableChecksPatternListModel: additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: - pattern_name = self.pattern_name + policy_name = self.policy_name priority = self.priority disabled = self.disabled description = self.description @@ -53,8 +53,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) - if pattern_name is not UNSET: - field_dict["pattern_name"] = pattern_name + if policy_name is not UNSET: + field_dict["policy_name"] = policy_name if priority is not UNSET: field_dict["priority"] = priority if disabled is not UNSET: @@ -75,7 +75,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.target_table_pattern_spec import TargetTablePatternSpec d = src_dict.copy() - pattern_name = d.pop("pattern_name", UNSET) + policy_name = d.pop("policy_name", UNSET) priority = d.pop("priority", UNSET) @@ -94,8 +94,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: yaml_parsing_error = d.pop("yaml_parsing_error", UNSET) - default_table_checks_pattern_list_model = cls( - pattern_name=pattern_name, + table_quality_policy_list_model = cls( + policy_name=policy_name, priority=priority, disabled=disabled, description=description, @@ -104,8 +104,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: yaml_parsing_error=yaml_parsing_error, ) - default_table_checks_pattern_list_model.additional_properties = d - return default_table_checks_pattern_list_model + table_quality_policy_list_model.additional_properties = d + return table_quality_policy_list_model @property def additional_keys(self) -> List[str]: diff --git a/distribution/python/dqops/client/models/default_column_checks_pattern_model.py b/distribution/python/dqops/client/models/table_quality_policy_model.py similarity index 57% rename from distribution/python/dqops/client/models/default_column_checks_pattern_model.py rename to distribution/python/dqops/client/models/table_quality_policy_model.py index 75ed3704b8..082f3c7656 100644 --- a/distribution/python/dqops/client/models/default_column_checks_pattern_model.py +++ b/distribution/python/dqops/client/models/table_quality_policy_model.py @@ -6,38 +6,36 @@ from ..types import UNSET, Unset if TYPE_CHECKING: - from ..models.column_default_checks_pattern_spec import ( - ColumnDefaultChecksPatternSpec, - ) + from ..models.table_quality_policy_spec import TableQualityPolicySpec -T = TypeVar("T", bound="DefaultColumnChecksPatternModel") +T = TypeVar("T", bound="TableQualityPolicyModel") @_attrs_define -class DefaultColumnChecksPatternModel: - """Default column-level checks pattern model +class TableQualityPolicyModel: + """Default table-level checks pattern (data quality policy) model Attributes: - pattern_name (Union[Unset, str]): Pattern name - pattern_spec (Union[Unset, ColumnDefaultChecksPatternSpec]): + policy_name (Union[Unset, str]): Quality policy name + policy_spec (Union[Unset, TableQualityPolicySpec]): can_edit (Union[Unset, bool]): Boolean flag that decides if the current user can update or delete this object. yaml_parsing_error (Union[Unset, str]): Optional parsing error that was captured when parsing the YAML file. This field is null when the YAML file is valid. If an error was captured, this field returns the file parsing error message and the file location. """ - pattern_name: Union[Unset, str] = UNSET - pattern_spec: Union[Unset, "ColumnDefaultChecksPatternSpec"] = UNSET + policy_name: Union[Unset, str] = UNSET + policy_spec: Union[Unset, "TableQualityPolicySpec"] = UNSET can_edit: Union[Unset, bool] = UNSET yaml_parsing_error: Union[Unset, str] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: - pattern_name = self.pattern_name - pattern_spec: Union[Unset, Dict[str, Any]] = UNSET - if not isinstance(self.pattern_spec, Unset): - pattern_spec = self.pattern_spec.to_dict() + policy_name = self.policy_name + policy_spec: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.policy_spec, Unset): + policy_spec = self.policy_spec.to_dict() can_edit = self.can_edit yaml_parsing_error = self.yaml_parsing_error @@ -45,10 +43,10 @@ def to_dict(self) -> Dict[str, Any]: field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) - if pattern_name is not UNSET: - field_dict["pattern_name"] = pattern_name - if pattern_spec is not UNSET: - field_dict["pattern_spec"] = pattern_spec + if policy_name is not UNSET: + field_dict["policy_name"] = policy_name + if policy_spec is not UNSET: + field_dict["policy_spec"] = policy_spec if can_edit is not UNSET: field_dict["can_edit"] = can_edit if yaml_parsing_error is not UNSET: @@ -58,33 +56,31 @@ def to_dict(self) -> Dict[str, Any]: @classmethod def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: - from ..models.column_default_checks_pattern_spec import ( - ColumnDefaultChecksPatternSpec, - ) + from ..models.table_quality_policy_spec import TableQualityPolicySpec d = src_dict.copy() - pattern_name = d.pop("pattern_name", UNSET) + policy_name = d.pop("policy_name", UNSET) - _pattern_spec = d.pop("pattern_spec", UNSET) - pattern_spec: Union[Unset, ColumnDefaultChecksPatternSpec] - if isinstance(_pattern_spec, Unset): - pattern_spec = UNSET + _policy_spec = d.pop("policy_spec", UNSET) + policy_spec: Union[Unset, TableQualityPolicySpec] + if isinstance(_policy_spec, Unset): + policy_spec = UNSET else: - pattern_spec = ColumnDefaultChecksPatternSpec.from_dict(_pattern_spec) + policy_spec = TableQualityPolicySpec.from_dict(_policy_spec) can_edit = d.pop("can_edit", UNSET) yaml_parsing_error = d.pop("yaml_parsing_error", UNSET) - default_column_checks_pattern_model = cls( - pattern_name=pattern_name, - pattern_spec=pattern_spec, + table_quality_policy_model = cls( + policy_name=policy_name, + policy_spec=policy_spec, can_edit=can_edit, yaml_parsing_error=yaml_parsing_error, ) - default_column_checks_pattern_model.additional_properties = d - return default_column_checks_pattern_model + table_quality_policy_model.additional_properties = d + return table_quality_policy_model @property def additional_keys(self) -> List[str]: diff --git a/distribution/python/dqops/client/models/table_default_checks_pattern_spec.py b/distribution/python/dqops/client/models/table_quality_policy_spec.py similarity index 96% rename from distribution/python/dqops/client/models/table_default_checks_pattern_spec.py rename to distribution/python/dqops/client/models/table_quality_policy_spec.py index bf32c2d0d5..8fb05e75ae 100644 --- a/distribution/python/dqops/client/models/table_default_checks_pattern_spec.py +++ b/distribution/python/dqops/client/models/table_quality_policy_spec.py @@ -18,11 +18,11 @@ from ..models.target_table_pattern_spec import TargetTablePatternSpec -T = TypeVar("T", bound="TableDefaultChecksPatternSpec") +T = TypeVar("T", bound="TableQualityPolicySpec") @_attrs_define -class TableDefaultChecksPatternSpec: +class TableQualityPolicySpec: """ Attributes: priority (Union[Unset, int]): The priority of the pattern. Patterns with lower values are applied before @@ -138,7 +138,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: _partitioned_checks ) - table_default_checks_pattern_spec = cls( + table_quality_policy_spec = cls( priority=priority, disabled=disabled, description=description, @@ -148,8 +148,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: partitioned_checks=partitioned_checks, ) - table_default_checks_pattern_spec.additional_properties = d - return table_default_checks_pattern_spec + table_quality_policy_spec.additional_properties = d + return table_quality_policy_spec @property def additional_keys(self) -> List[str]: diff --git a/distribution/python/dqops/client/models/table_spec.py b/distribution/python/dqops/client/models/table_spec.py index 2eb310f986..d0242b1d63 100644 --- a/distribution/python/dqops/client/models/table_spec.py +++ b/distribution/python/dqops/client/models/table_spec.py @@ -13,6 +13,7 @@ PartitionIncrementalTimeWindowSpec, ) from ..models.table_incident_grouping_spec import TableIncidentGroupingSpec + from ..models.table_lineage_source_spec import TableLineageSourceSpec from ..models.table_monitoring_check_categories_spec import ( TableMonitoringCheckCategoriesSpec, ) @@ -23,6 +24,7 @@ from ..models.table_profiling_check_categories_spec import ( TableProfilingCheckCategoriesSpec, ) + from ..models.table_spec_advanced_properties import TableSpecAdvancedProperties from ..models.table_spec_columns import TableSpecColumns from ..models.table_spec_groupings import TableSpecGroupings from ..models.table_spec_table_comparisons import TableSpecTableComparisons @@ -92,6 +94,10 @@ class TableSpec: comments (Union[Unset, List['CommentSpec']]): Comments used for change tracking and documenting changes directly in the table data quality specification file. file_format (Union[Unset, FileFormatSpec]): + advanced_properties (Union[Unset, TableSpecAdvancedProperties]): A dictionary of advanced properties that can be + used for e.g. to support mapping data to data catalogs, a key/value dictionary. + source_tables (Union[Unset, List['TableLineageSourceSpec']]): A list of source tables. This information is used + to define the data lineage report for the table. """ disabled: Union[Unset, bool] = UNSET @@ -116,6 +122,8 @@ class TableSpec: labels: Union[Unset, List[str]] = UNSET comments: Union[Unset, List["CommentSpec"]] = UNSET file_format: Union[Unset, "FileFormatSpec"] = UNSET + advanced_properties: Union[Unset, "TableSpecAdvancedProperties"] = UNSET + source_tables: Union[Unset, List["TableLineageSourceSpec"]] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -194,6 +202,18 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.file_format, Unset): file_format = self.file_format.to_dict() + advanced_properties: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.advanced_properties, Unset): + advanced_properties = self.advanced_properties.to_dict() + + source_tables: Union[Unset, List[Dict[str, Any]]] = UNSET + if not isinstance(self.source_tables, Unset): + source_tables = [] + for source_tables_item_data in self.source_tables: + source_tables_item = source_tables_item_data.to_dict() + + source_tables.append(source_tables_item) + field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -245,6 +265,10 @@ def to_dict(self) -> Dict[str, Any]: field_dict["comments"] = comments if file_format is not UNSET: field_dict["file_format"] = file_format + if advanced_properties is not UNSET: + field_dict["advanced_properties"] = advanced_properties + if source_tables is not UNSET: + field_dict["source_tables"] = source_tables return field_dict @@ -257,6 +281,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: PartitionIncrementalTimeWindowSpec, ) from ..models.table_incident_grouping_spec import TableIncidentGroupingSpec + from ..models.table_lineage_source_spec import TableLineageSourceSpec from ..models.table_monitoring_check_categories_spec import ( TableMonitoringCheckCategoriesSpec, ) @@ -267,6 +292,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: from ..models.table_profiling_check_categories_spec import ( TableProfilingCheckCategoriesSpec, ) + from ..models.table_spec_advanced_properties import TableSpecAdvancedProperties from ..models.table_spec_columns import TableSpecColumns from ..models.table_spec_groupings import TableSpecGroupings from ..models.table_spec_table_comparisons import TableSpecTableComparisons @@ -404,6 +430,24 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: file_format = FileFormatSpec.from_dict(_file_format) + _advanced_properties = d.pop("advanced_properties", UNSET) + advanced_properties: Union[Unset, TableSpecAdvancedProperties] + if isinstance(_advanced_properties, Unset): + advanced_properties = UNSET + else: + advanced_properties = TableSpecAdvancedProperties.from_dict( + _advanced_properties + ) + + source_tables = [] + _source_tables = d.pop("source_tables", UNSET) + for source_tables_item_data in _source_tables or []: + source_tables_item = TableLineageSourceSpec.from_dict( + source_tables_item_data + ) + + source_tables.append(source_tables_item) + table_spec = cls( disabled=disabled, stage=stage, @@ -427,6 +471,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: labels=labels, comments=comments, file_format=file_format, + advanced_properties=advanced_properties, + source_tables=source_tables, ) table_spec.additional_properties = d diff --git a/distribution/python/dqops/client/models/table_spec_advanced_properties.py b/distribution/python/dqops/client/models/table_spec_advanced_properties.py new file mode 100644 index 0000000000..0c607eab9b --- /dev/null +++ b/distribution/python/dqops/client/models/table_spec_advanced_properties.py @@ -0,0 +1,48 @@ +from typing import Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +T = TypeVar("T", bound="TableSpecAdvancedProperties") + + +@_attrs_define +class TableSpecAdvancedProperties: + """A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value + dictionary. + + """ + + additional_properties: Dict[str, str] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + d = src_dict.copy() + table_spec_advanced_properties = cls() + + table_spec_advanced_properties.additional_properties = d + return table_spec_advanced_properties + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> str: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: str) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_daily_monitoring_checks_spec.py b/distribution/python/dqops/client/models/table_uniqueness_daily_monitoring_checks_spec.py new file mode 100644 index 0000000000..900a307eaf --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_daily_monitoring_checks_spec.py @@ -0,0 +1,145 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_daily_monitoring_checks_spec_custom_checks import ( + TableUniquenessDailyMonitoringChecksSpecCustomChecks, + ) + + +T = TypeVar("T", bound="TableUniquenessDailyMonitoringChecksSpec") + + +@_attrs_define +class TableUniquenessDailyMonitoringChecksSpec: + """ + Attributes: + custom_checks (Union[Unset, TableUniquenessDailyMonitoringChecksSpecCustomChecks]): Dictionary of additional + custom checks within this category. The keys are check names defined in the definition section. The sensor + parameters and rules should match the type of the configured sensor and rule for the custom check. + daily_duplicate_record_count (Union[Unset, TableDuplicateRecordCountCheckSpec]): + daily_duplicate_record_percent (Union[Unset, TableDuplicateRecordPercentCheckSpec]): + """ + + custom_checks: Union[ + Unset, "TableUniquenessDailyMonitoringChecksSpecCustomChecks" + ] = UNSET + daily_duplicate_record_count: Union[Unset, "TableDuplicateRecordCountCheckSpec"] = ( + UNSET + ) + daily_duplicate_record_percent: Union[ + Unset, "TableDuplicateRecordPercentCheckSpec" + ] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + custom_checks: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.custom_checks, Unset): + custom_checks = self.custom_checks.to_dict() + + daily_duplicate_record_count: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.daily_duplicate_record_count, Unset): + daily_duplicate_record_count = self.daily_duplicate_record_count.to_dict() + + daily_duplicate_record_percent: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.daily_duplicate_record_percent, Unset): + daily_duplicate_record_percent = ( + self.daily_duplicate_record_percent.to_dict() + ) + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if custom_checks is not UNSET: + field_dict["custom_checks"] = custom_checks + if daily_duplicate_record_count is not UNSET: + field_dict["daily_duplicate_record_count"] = daily_duplicate_record_count + if daily_duplicate_record_percent is not UNSET: + field_dict["daily_duplicate_record_percent"] = ( + daily_duplicate_record_percent + ) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_daily_monitoring_checks_spec_custom_checks import ( + TableUniquenessDailyMonitoringChecksSpecCustomChecks, + ) + + d = src_dict.copy() + _custom_checks = d.pop("custom_checks", UNSET) + custom_checks: Union[ + Unset, TableUniquenessDailyMonitoringChecksSpecCustomChecks + ] + if isinstance(_custom_checks, Unset): + custom_checks = UNSET + else: + custom_checks = ( + TableUniquenessDailyMonitoringChecksSpecCustomChecks.from_dict( + _custom_checks + ) + ) + + _daily_duplicate_record_count = d.pop("daily_duplicate_record_count", UNSET) + daily_duplicate_record_count: Union[Unset, TableDuplicateRecordCountCheckSpec] + if isinstance(_daily_duplicate_record_count, Unset): + daily_duplicate_record_count = UNSET + else: + daily_duplicate_record_count = TableDuplicateRecordCountCheckSpec.from_dict( + _daily_duplicate_record_count + ) + + _daily_duplicate_record_percent = d.pop("daily_duplicate_record_percent", UNSET) + daily_duplicate_record_percent: Union[ + Unset, TableDuplicateRecordPercentCheckSpec + ] + if isinstance(_daily_duplicate_record_percent, Unset): + daily_duplicate_record_percent = UNSET + else: + daily_duplicate_record_percent = ( + TableDuplicateRecordPercentCheckSpec.from_dict( + _daily_duplicate_record_percent + ) + ) + + table_uniqueness_daily_monitoring_checks_spec = cls( + custom_checks=custom_checks, + daily_duplicate_record_count=daily_duplicate_record_count, + daily_duplicate_record_percent=daily_duplicate_record_percent, + ) + + table_uniqueness_daily_monitoring_checks_spec.additional_properties = d + return table_uniqueness_daily_monitoring_checks_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_daily_monitoring_checks_spec_custom_checks.py b/distribution/python/dqops/client/models/table_uniqueness_daily_monitoring_checks_spec_custom_checks.py new file mode 100644 index 0000000000..f888a072fa --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_daily_monitoring_checks_spec_custom_checks.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.custom_check_spec import CustomCheckSpec + + +T = TypeVar("T", bound="TableUniquenessDailyMonitoringChecksSpecCustomChecks") + + +@_attrs_define +class TableUniquenessDailyMonitoringChecksSpecCustomChecks: + """Dictionary of additional custom checks within this category. The keys are check names defined in the definition + section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom + check. + + """ + + additional_properties: Dict[str, "CustomCheckSpec"] = _attrs_field( + init=False, factory=dict + ) + + def to_dict(self) -> Dict[str, Any]: + pass + + field_dict: Dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.custom_check_spec import CustomCheckSpec + + d = src_dict.copy() + table_uniqueness_daily_monitoring_checks_spec_custom_checks = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = CustomCheckSpec.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + table_uniqueness_daily_monitoring_checks_spec_custom_checks.additional_properties = ( + additional_properties + ) + return table_uniqueness_daily_monitoring_checks_spec_custom_checks + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> "CustomCheckSpec": + return self.additional_properties[key] + + def __setitem__(self, key: str, value: "CustomCheckSpec") -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_daily_partition_checks_spec.py b/distribution/python/dqops/client/models/table_uniqueness_daily_partition_checks_spec.py new file mode 100644 index 0000000000..e5ed88c9f9 --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_daily_partition_checks_spec.py @@ -0,0 +1,155 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_daily_partition_checks_spec_custom_checks import ( + TableUniquenessDailyPartitionChecksSpecCustomChecks, + ) + + +T = TypeVar("T", bound="TableUniquenessDailyPartitionChecksSpec") + + +@_attrs_define +class TableUniquenessDailyPartitionChecksSpec: + """ + Attributes: + custom_checks (Union[Unset, TableUniquenessDailyPartitionChecksSpecCustomChecks]): Dictionary of additional + custom checks within this category. The keys are check names defined in the definition section. The sensor + parameters and rules should match the type of the configured sensor and rule for the custom check. + daily_partition_duplicate_record_count (Union[Unset, TableDuplicateRecordCountCheckSpec]): + daily_partition_duplicate_record_percent (Union[Unset, TableDuplicateRecordPercentCheckSpec]): + """ + + custom_checks: Union[ + Unset, "TableUniquenessDailyPartitionChecksSpecCustomChecks" + ] = UNSET + daily_partition_duplicate_record_count: Union[ + Unset, "TableDuplicateRecordCountCheckSpec" + ] = UNSET + daily_partition_duplicate_record_percent: Union[ + Unset, "TableDuplicateRecordPercentCheckSpec" + ] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + custom_checks: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.custom_checks, Unset): + custom_checks = self.custom_checks.to_dict() + + daily_partition_duplicate_record_count: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.daily_partition_duplicate_record_count, Unset): + daily_partition_duplicate_record_count = ( + self.daily_partition_duplicate_record_count.to_dict() + ) + + daily_partition_duplicate_record_percent: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.daily_partition_duplicate_record_percent, Unset): + daily_partition_duplicate_record_percent = ( + self.daily_partition_duplicate_record_percent.to_dict() + ) + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if custom_checks is not UNSET: + field_dict["custom_checks"] = custom_checks + if daily_partition_duplicate_record_count is not UNSET: + field_dict["daily_partition_duplicate_record_count"] = ( + daily_partition_duplicate_record_count + ) + if daily_partition_duplicate_record_percent is not UNSET: + field_dict["daily_partition_duplicate_record_percent"] = ( + daily_partition_duplicate_record_percent + ) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_daily_partition_checks_spec_custom_checks import ( + TableUniquenessDailyPartitionChecksSpecCustomChecks, + ) + + d = src_dict.copy() + _custom_checks = d.pop("custom_checks", UNSET) + custom_checks: Union[Unset, TableUniquenessDailyPartitionChecksSpecCustomChecks] + if isinstance(_custom_checks, Unset): + custom_checks = UNSET + else: + custom_checks = ( + TableUniquenessDailyPartitionChecksSpecCustomChecks.from_dict( + _custom_checks + ) + ) + + _daily_partition_duplicate_record_count = d.pop( + "daily_partition_duplicate_record_count", UNSET + ) + daily_partition_duplicate_record_count: Union[ + Unset, TableDuplicateRecordCountCheckSpec + ] + if isinstance(_daily_partition_duplicate_record_count, Unset): + daily_partition_duplicate_record_count = UNSET + else: + daily_partition_duplicate_record_count = ( + TableDuplicateRecordCountCheckSpec.from_dict( + _daily_partition_duplicate_record_count + ) + ) + + _daily_partition_duplicate_record_percent = d.pop( + "daily_partition_duplicate_record_percent", UNSET + ) + daily_partition_duplicate_record_percent: Union[ + Unset, TableDuplicateRecordPercentCheckSpec + ] + if isinstance(_daily_partition_duplicate_record_percent, Unset): + daily_partition_duplicate_record_percent = UNSET + else: + daily_partition_duplicate_record_percent = ( + TableDuplicateRecordPercentCheckSpec.from_dict( + _daily_partition_duplicate_record_percent + ) + ) + + table_uniqueness_daily_partition_checks_spec = cls( + custom_checks=custom_checks, + daily_partition_duplicate_record_count=daily_partition_duplicate_record_count, + daily_partition_duplicate_record_percent=daily_partition_duplicate_record_percent, + ) + + table_uniqueness_daily_partition_checks_spec.additional_properties = d + return table_uniqueness_daily_partition_checks_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_daily_partition_checks_spec_custom_checks.py b/distribution/python/dqops/client/models/table_uniqueness_daily_partition_checks_spec_custom_checks.py new file mode 100644 index 0000000000..2800661a08 --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_daily_partition_checks_spec_custom_checks.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.custom_check_spec import CustomCheckSpec + + +T = TypeVar("T", bound="TableUniquenessDailyPartitionChecksSpecCustomChecks") + + +@_attrs_define +class TableUniquenessDailyPartitionChecksSpecCustomChecks: + """Dictionary of additional custom checks within this category. The keys are check names defined in the definition + section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom + check. + + """ + + additional_properties: Dict[str, "CustomCheckSpec"] = _attrs_field( + init=False, factory=dict + ) + + def to_dict(self) -> Dict[str, Any]: + pass + + field_dict: Dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.custom_check_spec import CustomCheckSpec + + d = src_dict.copy() + table_uniqueness_daily_partition_checks_spec_custom_checks = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = CustomCheckSpec.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + table_uniqueness_daily_partition_checks_spec_custom_checks.additional_properties = ( + additional_properties + ) + return table_uniqueness_daily_partition_checks_spec_custom_checks + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> "CustomCheckSpec": + return self.additional_properties[key] + + def __setitem__(self, key: str, value: "CustomCheckSpec") -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_monthly_monitoring_checks_spec.py b/distribution/python/dqops/client/models/table_uniqueness_monthly_monitoring_checks_spec.py new file mode 100644 index 0000000000..5252fcfd07 --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_monthly_monitoring_checks_spec.py @@ -0,0 +1,153 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_monthly_monitoring_checks_spec_custom_checks import ( + TableUniquenessMonthlyMonitoringChecksSpecCustomChecks, + ) + + +T = TypeVar("T", bound="TableUniquenessMonthlyMonitoringChecksSpec") + + +@_attrs_define +class TableUniquenessMonthlyMonitoringChecksSpec: + """ + Attributes: + custom_checks (Union[Unset, TableUniquenessMonthlyMonitoringChecksSpecCustomChecks]): Dictionary of additional + custom checks within this category. The keys are check names defined in the definition section. The sensor + parameters and rules should match the type of the configured sensor and rule for the custom check. + monthly_duplicate_record_count (Union[Unset, TableDuplicateRecordCountCheckSpec]): + monthly_duplicate_record_percent (Union[Unset, TableDuplicateRecordPercentCheckSpec]): + """ + + custom_checks: Union[ + Unset, "TableUniquenessMonthlyMonitoringChecksSpecCustomChecks" + ] = UNSET + monthly_duplicate_record_count: Union[ + Unset, "TableDuplicateRecordCountCheckSpec" + ] = UNSET + monthly_duplicate_record_percent: Union[ + Unset, "TableDuplicateRecordPercentCheckSpec" + ] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + custom_checks: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.custom_checks, Unset): + custom_checks = self.custom_checks.to_dict() + + monthly_duplicate_record_count: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.monthly_duplicate_record_count, Unset): + monthly_duplicate_record_count = ( + self.monthly_duplicate_record_count.to_dict() + ) + + monthly_duplicate_record_percent: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.monthly_duplicate_record_percent, Unset): + monthly_duplicate_record_percent = ( + self.monthly_duplicate_record_percent.to_dict() + ) + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if custom_checks is not UNSET: + field_dict["custom_checks"] = custom_checks + if monthly_duplicate_record_count is not UNSET: + field_dict["monthly_duplicate_record_count"] = ( + monthly_duplicate_record_count + ) + if monthly_duplicate_record_percent is not UNSET: + field_dict["monthly_duplicate_record_percent"] = ( + monthly_duplicate_record_percent + ) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_monthly_monitoring_checks_spec_custom_checks import ( + TableUniquenessMonthlyMonitoringChecksSpecCustomChecks, + ) + + d = src_dict.copy() + _custom_checks = d.pop("custom_checks", UNSET) + custom_checks: Union[ + Unset, TableUniquenessMonthlyMonitoringChecksSpecCustomChecks + ] + if isinstance(_custom_checks, Unset): + custom_checks = UNSET + else: + custom_checks = ( + TableUniquenessMonthlyMonitoringChecksSpecCustomChecks.from_dict( + _custom_checks + ) + ) + + _monthly_duplicate_record_count = d.pop("monthly_duplicate_record_count", UNSET) + monthly_duplicate_record_count: Union[Unset, TableDuplicateRecordCountCheckSpec] + if isinstance(_monthly_duplicate_record_count, Unset): + monthly_duplicate_record_count = UNSET + else: + monthly_duplicate_record_count = ( + TableDuplicateRecordCountCheckSpec.from_dict( + _monthly_duplicate_record_count + ) + ) + + _monthly_duplicate_record_percent = d.pop( + "monthly_duplicate_record_percent", UNSET + ) + monthly_duplicate_record_percent: Union[ + Unset, TableDuplicateRecordPercentCheckSpec + ] + if isinstance(_monthly_duplicate_record_percent, Unset): + monthly_duplicate_record_percent = UNSET + else: + monthly_duplicate_record_percent = ( + TableDuplicateRecordPercentCheckSpec.from_dict( + _monthly_duplicate_record_percent + ) + ) + + table_uniqueness_monthly_monitoring_checks_spec = cls( + custom_checks=custom_checks, + monthly_duplicate_record_count=monthly_duplicate_record_count, + monthly_duplicate_record_percent=monthly_duplicate_record_percent, + ) + + table_uniqueness_monthly_monitoring_checks_spec.additional_properties = d + return table_uniqueness_monthly_monitoring_checks_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_monthly_monitoring_checks_spec_custom_checks.py b/distribution/python/dqops/client/models/table_uniqueness_monthly_monitoring_checks_spec_custom_checks.py new file mode 100644 index 0000000000..eb5d5c2b0a --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_monthly_monitoring_checks_spec_custom_checks.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.custom_check_spec import CustomCheckSpec + + +T = TypeVar("T", bound="TableUniquenessMonthlyMonitoringChecksSpecCustomChecks") + + +@_attrs_define +class TableUniquenessMonthlyMonitoringChecksSpecCustomChecks: + """Dictionary of additional custom checks within this category. The keys are check names defined in the definition + section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom + check. + + """ + + additional_properties: Dict[str, "CustomCheckSpec"] = _attrs_field( + init=False, factory=dict + ) + + def to_dict(self) -> Dict[str, Any]: + pass + + field_dict: Dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.custom_check_spec import CustomCheckSpec + + d = src_dict.copy() + table_uniqueness_monthly_monitoring_checks_spec_custom_checks = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = CustomCheckSpec.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + table_uniqueness_monthly_monitoring_checks_spec_custom_checks.additional_properties = ( + additional_properties + ) + return table_uniqueness_monthly_monitoring_checks_spec_custom_checks + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> "CustomCheckSpec": + return self.additional_properties[key] + + def __setitem__(self, key: str, value: "CustomCheckSpec") -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_monthly_partition_checks_spec.py b/distribution/python/dqops/client/models/table_uniqueness_monthly_partition_checks_spec.py new file mode 100644 index 0000000000..b6610e37cf --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_monthly_partition_checks_spec.py @@ -0,0 +1,157 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_monthly_partition_checks_spec_custom_checks import ( + TableUniquenessMonthlyPartitionChecksSpecCustomChecks, + ) + + +T = TypeVar("T", bound="TableUniquenessMonthlyPartitionChecksSpec") + + +@_attrs_define +class TableUniquenessMonthlyPartitionChecksSpec: + """ + Attributes: + custom_checks (Union[Unset, TableUniquenessMonthlyPartitionChecksSpecCustomChecks]): Dictionary of additional + custom checks within this category. The keys are check names defined in the definition section. The sensor + parameters and rules should match the type of the configured sensor and rule for the custom check. + monthly_partition_duplicate_record_count (Union[Unset, TableDuplicateRecordCountCheckSpec]): + monthly_partition_duplicate_record_percent (Union[Unset, TableDuplicateRecordPercentCheckSpec]): + """ + + custom_checks: Union[ + Unset, "TableUniquenessMonthlyPartitionChecksSpecCustomChecks" + ] = UNSET + monthly_partition_duplicate_record_count: Union[ + Unset, "TableDuplicateRecordCountCheckSpec" + ] = UNSET + monthly_partition_duplicate_record_percent: Union[ + Unset, "TableDuplicateRecordPercentCheckSpec" + ] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + custom_checks: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.custom_checks, Unset): + custom_checks = self.custom_checks.to_dict() + + monthly_partition_duplicate_record_count: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.monthly_partition_duplicate_record_count, Unset): + monthly_partition_duplicate_record_count = ( + self.monthly_partition_duplicate_record_count.to_dict() + ) + + monthly_partition_duplicate_record_percent: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.monthly_partition_duplicate_record_percent, Unset): + monthly_partition_duplicate_record_percent = ( + self.monthly_partition_duplicate_record_percent.to_dict() + ) + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if custom_checks is not UNSET: + field_dict["custom_checks"] = custom_checks + if monthly_partition_duplicate_record_count is not UNSET: + field_dict["monthly_partition_duplicate_record_count"] = ( + monthly_partition_duplicate_record_count + ) + if monthly_partition_duplicate_record_percent is not UNSET: + field_dict["monthly_partition_duplicate_record_percent"] = ( + monthly_partition_duplicate_record_percent + ) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_monthly_partition_checks_spec_custom_checks import ( + TableUniquenessMonthlyPartitionChecksSpecCustomChecks, + ) + + d = src_dict.copy() + _custom_checks = d.pop("custom_checks", UNSET) + custom_checks: Union[ + Unset, TableUniquenessMonthlyPartitionChecksSpecCustomChecks + ] + if isinstance(_custom_checks, Unset): + custom_checks = UNSET + else: + custom_checks = ( + TableUniquenessMonthlyPartitionChecksSpecCustomChecks.from_dict( + _custom_checks + ) + ) + + _monthly_partition_duplicate_record_count = d.pop( + "monthly_partition_duplicate_record_count", UNSET + ) + monthly_partition_duplicate_record_count: Union[ + Unset, TableDuplicateRecordCountCheckSpec + ] + if isinstance(_monthly_partition_duplicate_record_count, Unset): + monthly_partition_duplicate_record_count = UNSET + else: + monthly_partition_duplicate_record_count = ( + TableDuplicateRecordCountCheckSpec.from_dict( + _monthly_partition_duplicate_record_count + ) + ) + + _monthly_partition_duplicate_record_percent = d.pop( + "monthly_partition_duplicate_record_percent", UNSET + ) + monthly_partition_duplicate_record_percent: Union[ + Unset, TableDuplicateRecordPercentCheckSpec + ] + if isinstance(_monthly_partition_duplicate_record_percent, Unset): + monthly_partition_duplicate_record_percent = UNSET + else: + monthly_partition_duplicate_record_percent = ( + TableDuplicateRecordPercentCheckSpec.from_dict( + _monthly_partition_duplicate_record_percent + ) + ) + + table_uniqueness_monthly_partition_checks_spec = cls( + custom_checks=custom_checks, + monthly_partition_duplicate_record_count=monthly_partition_duplicate_record_count, + monthly_partition_duplicate_record_percent=monthly_partition_duplicate_record_percent, + ) + + table_uniqueness_monthly_partition_checks_spec.additional_properties = d + return table_uniqueness_monthly_partition_checks_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_monthly_partition_checks_spec_custom_checks.py b/distribution/python/dqops/client/models/table_uniqueness_monthly_partition_checks_spec_custom_checks.py new file mode 100644 index 0000000000..cd3e136af9 --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_monthly_partition_checks_spec_custom_checks.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.custom_check_spec import CustomCheckSpec + + +T = TypeVar("T", bound="TableUniquenessMonthlyPartitionChecksSpecCustomChecks") + + +@_attrs_define +class TableUniquenessMonthlyPartitionChecksSpecCustomChecks: + """Dictionary of additional custom checks within this category. The keys are check names defined in the definition + section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom + check. + + """ + + additional_properties: Dict[str, "CustomCheckSpec"] = _attrs_field( + init=False, factory=dict + ) + + def to_dict(self) -> Dict[str, Any]: + pass + + field_dict: Dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.custom_check_spec import CustomCheckSpec + + d = src_dict.copy() + table_uniqueness_monthly_partition_checks_spec_custom_checks = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = CustomCheckSpec.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + table_uniqueness_monthly_partition_checks_spec_custom_checks.additional_properties = ( + additional_properties + ) + return table_uniqueness_monthly_partition_checks_spec_custom_checks + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> "CustomCheckSpec": + return self.additional_properties[key] + + def __setitem__(self, key: str, value: "CustomCheckSpec") -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_profiling_checks_spec.py b/distribution/python/dqops/client/models/table_uniqueness_profiling_checks_spec.py new file mode 100644 index 0000000000..cca1ed502a --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_profiling_checks_spec.py @@ -0,0 +1,149 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_profiling_checks_spec_custom_checks import ( + TableUniquenessProfilingChecksSpecCustomChecks, + ) + + +T = TypeVar("T", bound="TableUniquenessProfilingChecksSpec") + + +@_attrs_define +class TableUniquenessProfilingChecksSpec: + """ + Attributes: + custom_checks (Union[Unset, TableUniquenessProfilingChecksSpecCustomChecks]): Dictionary of additional custom + checks within this category. The keys are check names defined in the definition section. The sensor parameters + and rules should match the type of the configured sensor and rule for the custom check. + profile_duplicate_record_count (Union[Unset, TableDuplicateRecordCountCheckSpec]): + profile_duplicate_record_percent (Union[Unset, TableDuplicateRecordPercentCheckSpec]): + """ + + custom_checks: Union[Unset, "TableUniquenessProfilingChecksSpecCustomChecks"] = ( + UNSET + ) + profile_duplicate_record_count: Union[ + Unset, "TableDuplicateRecordCountCheckSpec" + ] = UNSET + profile_duplicate_record_percent: Union[ + Unset, "TableDuplicateRecordPercentCheckSpec" + ] = UNSET + additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> Dict[str, Any]: + custom_checks: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.custom_checks, Unset): + custom_checks = self.custom_checks.to_dict() + + profile_duplicate_record_count: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.profile_duplicate_record_count, Unset): + profile_duplicate_record_count = ( + self.profile_duplicate_record_count.to_dict() + ) + + profile_duplicate_record_percent: Union[Unset, Dict[str, Any]] = UNSET + if not isinstance(self.profile_duplicate_record_percent, Unset): + profile_duplicate_record_percent = ( + self.profile_duplicate_record_percent.to_dict() + ) + + field_dict: Dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if custom_checks is not UNSET: + field_dict["custom_checks"] = custom_checks + if profile_duplicate_record_count is not UNSET: + field_dict["profile_duplicate_record_count"] = ( + profile_duplicate_record_count + ) + if profile_duplicate_record_percent is not UNSET: + field_dict["profile_duplicate_record_percent"] = ( + profile_duplicate_record_percent + ) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.table_duplicate_record_count_check_spec import ( + TableDuplicateRecordCountCheckSpec, + ) + from ..models.table_duplicate_record_percent_check_spec import ( + TableDuplicateRecordPercentCheckSpec, + ) + from ..models.table_uniqueness_profiling_checks_spec_custom_checks import ( + TableUniquenessProfilingChecksSpecCustomChecks, + ) + + d = src_dict.copy() + _custom_checks = d.pop("custom_checks", UNSET) + custom_checks: Union[Unset, TableUniquenessProfilingChecksSpecCustomChecks] + if isinstance(_custom_checks, Unset): + custom_checks = UNSET + else: + custom_checks = TableUniquenessProfilingChecksSpecCustomChecks.from_dict( + _custom_checks + ) + + _profile_duplicate_record_count = d.pop("profile_duplicate_record_count", UNSET) + profile_duplicate_record_count: Union[Unset, TableDuplicateRecordCountCheckSpec] + if isinstance(_profile_duplicate_record_count, Unset): + profile_duplicate_record_count = UNSET + else: + profile_duplicate_record_count = ( + TableDuplicateRecordCountCheckSpec.from_dict( + _profile_duplicate_record_count + ) + ) + + _profile_duplicate_record_percent = d.pop( + "profile_duplicate_record_percent", UNSET + ) + profile_duplicate_record_percent: Union[ + Unset, TableDuplicateRecordPercentCheckSpec + ] + if isinstance(_profile_duplicate_record_percent, Unset): + profile_duplicate_record_percent = UNSET + else: + profile_duplicate_record_percent = ( + TableDuplicateRecordPercentCheckSpec.from_dict( + _profile_duplicate_record_percent + ) + ) + + table_uniqueness_profiling_checks_spec = cls( + custom_checks=custom_checks, + profile_duplicate_record_count=profile_duplicate_record_count, + profile_duplicate_record_percent=profile_duplicate_record_percent, + ) + + table_uniqueness_profiling_checks_spec.additional_properties = d + return table_uniqueness_profiling_checks_spec + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/table_uniqueness_profiling_checks_spec_custom_checks.py b/distribution/python/dqops/client/models/table_uniqueness_profiling_checks_spec_custom_checks.py new file mode 100644 index 0000000000..0e9d8f88ef --- /dev/null +++ b/distribution/python/dqops/client/models/table_uniqueness_profiling_checks_spec_custom_checks.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.custom_check_spec import CustomCheckSpec + + +T = TypeVar("T", bound="TableUniquenessProfilingChecksSpecCustomChecks") + + +@_attrs_define +class TableUniquenessProfilingChecksSpecCustomChecks: + """Dictionary of additional custom checks within this category. The keys are check names defined in the definition + section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom + check. + + """ + + additional_properties: Dict[str, "CustomCheckSpec"] = _attrs_field( + init=False, factory=dict + ) + + def to_dict(self) -> Dict[str, Any]: + pass + + field_dict: Dict[str, Any] = {} + for prop_name, prop in self.additional_properties.items(): + field_dict[prop_name] = prop.to_dict() + + field_dict.update({}) + + return field_dict + + @classmethod + def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: + from ..models.custom_check_spec import CustomCheckSpec + + d = src_dict.copy() + table_uniqueness_profiling_checks_spec_custom_checks = cls() + + additional_properties = {} + for prop_name, prop_dict in d.items(): + additional_property = CustomCheckSpec.from_dict(prop_dict) + + additional_properties[prop_name] = additional_property + + table_uniqueness_profiling_checks_spec_custom_checks.additional_properties = ( + additional_properties + ) + return table_uniqueness_profiling_checks_spec_custom_checks + + @property + def additional_keys(self) -> List[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> "CustomCheckSpec": + return self.additional_properties[key] + + def __setitem__(self, key: str, value: "CustomCheckSpec") -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/distribution/python/dqops/client/models/time_window_filter_parameters.py b/distribution/python/dqops/client/models/time_window_filter_parameters.py index 806165538e..64eeca7879 100644 --- a/distribution/python/dqops/client/models/time_window_filter_parameters.py +++ b/distribution/python/dqops/client/models/time_window_filter_parameters.py @@ -52,6 +52,13 @@ class TimeWindowFilterParameters: offset (yyyy-MM-dd HH:mm:ss). For example: 2023-02-20 14:10:00+02. The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date and time overrides the parameters to disable analyzing today or the current month. + where_filter (Union[Unset, str]): An additional filter which must be a valid SQL predicate (an SQL expression + that returns 'true' or 'false') that is added to the WHERE clause of the SQL query that DQOps will run on the + data source. The purpose of a custom filter is to analyze only a subset of data, for example, when a new batch + of records is loaded, and the data quality checks are evaluated as a data contract. All the records in that + batch must tagged with the same value, and the passed predicate to find records from that batch would use the + filter in the form: "{alias}.batch_id = 1". The filter can use replacement tokens {alias} to reference the + analyzed table. """ daily_partitioning_recent_days: Union[Unset, int] = UNSET @@ -64,6 +71,7 @@ class TimeWindowFilterParameters: to_date: Union[Unset, datetime.date] = UNSET to_date_time: Union[Unset, datetime.datetime] = UNSET to_date_time_offset: Union[Unset, datetime.datetime] = UNSET + where_filter: Union[Unset, str] = UNSET additional_properties: Dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> Dict[str, Any]: @@ -97,6 +105,8 @@ def to_dict(self) -> Dict[str, Any]: if not isinstance(self.to_date_time_offset, Unset): to_date_time_offset = self.to_date_time_offset.isoformat() + where_filter = self.where_filter + field_dict: Dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update({}) @@ -128,6 +138,8 @@ def to_dict(self) -> Dict[str, Any]: field_dict["to_date_time"] = to_date_time if to_date_time_offset is not UNSET: field_dict["to_date_time_offset"] = to_date_time_offset + if where_filter is not UNSET: + field_dict["where_filter"] = where_filter return field_dict @@ -190,6 +202,8 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: else: to_date_time_offset = isoparse(_to_date_time_offset) + where_filter = d.pop("where_filter", UNSET) + time_window_filter_parameters = cls( daily_partitioning_recent_days=daily_partitioning_recent_days, daily_partitioning_include_today=daily_partitioning_include_today, @@ -201,6 +215,7 @@ def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T: to_date=to_date, to_date_time=to_date_time, to_date_time_offset=to_date_time_offset, + where_filter=where_filter, ) time_window_filter_parameters.additional_properties = d diff --git a/docs/categories-of-data-quality-checks/how-to-detect-anomaly-data-quality-issues.md b/docs/categories-of-data-quality-checks/how-to-detect-anomaly-data-quality-issues.md index 5c81fb4bec..6d301f18ab 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-anomaly-data-quality-issues.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-anomaly-data-quality-issues.md @@ -81,7 +81,7 @@ The chart view shows all anomalies of the maximum value in the latitude column. To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. For the Partition checks, samples are collected from entire table, not specific partition. +**Error sampling** tab in the results section. For the Partition checks, samples are collected from entire table, not specific partition. ![Data anomalies in a mean value of latitude in partitions chart view - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/numeric-mean-anomaly-partitions-chart-error-sampling.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md b/docs/categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md index 6a7abaa3c9..abc22582bc 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md @@ -31,39 +31,49 @@ We can also validate fields containing auto-generated identifiers, such as: - Tax identifiers. DQOps has built-in data quality checks for validating typical text formats. -Validation of non-standard text values is supported by using a [custom regular expression](../checks/column/patterns/texts-matching-regex-percent.md) +Validation of non-standard text values is supported by using a [custom regular expression](../checks/column/patterns/texts-not-matching-regex-percent.md) or defining a [custom data quality check](../working-with-dqo/creating-custom-data-quality-checks.md). ## Built-in patterns The DQOps data quality checks that detect the most popular patterns are listed in the table below. -| Data quality check name | Description | Sample texts | -|-------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -| [*invalid_email_format_found*](../checks/column/patterns/invalid-email-format-found.md) | This check finds texts that are not valid emails. | *john.smith@example.com* | -| [*text_not_matching_date_pattern_found*](../checks/column/patterns/text-not-matching-date-pattern-found.md) | This check measures the percentage of texts that match a specified date format. The supported patterns are: *"YYYY-MM-DD"*, *"MM/DD/YYYY*", *"DD/MM/YYYY"*, *"YYYY/MM/DD"*, *"Month D, YYYY"*. | *2024-02-10*, *02/10/2024* | -| [*text_matching_date_pattern_percent*](../checks/column/patterns/text-matching-date-pattern-percent.md) | This check measures the percentage of texts that match a specified date format. | *2024-02-10*, *02/10/2024* | -| [*text_matching_name_pattern_percent*](../checks/column/patterns/text-matching-name-pattern-percent.md) | This check validates text values that are valid identifiers and measures the percentage of valid values. A valid identifier is a text that contains only letters. | *abcd* | -| [*invalid_uuid_format_found*](../checks/column/patterns/invalid-uuid-format-found.md) | This check validates common UUID and GUID formats and finds any values that are not in the correct format. | *46f6c2f8-038f-4f36-9b87-fbf3fcd341cc* | -| [*valid_uuid_format_percent*](../checks/column/patterns/valid-uuid-format-percent.md) | This check validates common UUID and GUID formats and measures the percentage of valid values. | *46f6c2f8-038f-4f36-9b87-fbf3fcd341cc* | -| [*invalid_ip4_address_format_found*](../checks/column/patterns/invalid-ip4-address-format-found.md) | This check validates the format of IP4 addresses and detects invalid values. | *123.45.67.89* | -| [*invalid_ip6_address_format_found*](../checks/column/patterns/invalid-ip6-address-format-found.md) | This check validates the format of IP6 addresses and detects invalid values. | *66b2:454b:2638:9d9e:b0c1:1c3e:a39c:e83a* | +| Data quality check name | Description | Sample texts | +|-----------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| +| [*invalid_email_format_found*](../checks/column/patterns/invalid-email-format-found.md) | This check finds texts that are not valid emails. | *john.smith@example.com* | +| [*text_not_matching_date_pattern_found*](../checks/column/patterns/text-not-matching-date-pattern-found.md) | This check measures the percentage of texts that do not match a specified date format. The supported patterns are: *"YYYY-MM-DD"*, *"MM/DD/YYYY*", *"DD/MM/YYYY"*, *"YYYY/MM/DD"*, *"Month D, YYYY"*. | *2024-02-10*, *02/10/2024* | +| [*text_not_matching_date_pattern_percent*](../checks/column/patterns/text-not-matching-date-pattern-percent.md) | This check measures the percentage of texts that do not match a specified date format. | *2024-02-10*, *02/10/2024* | +| [*text_not_matching_name_pattern_percent*](../checks/column/patterns/text-not-matching-name-pattern-percent.md) | This check validates text values that are valid identifiers and measures the percentage of invalid values. A valid identifier is a text that contains only letters. | *abcd* | +| [*invalid_uuid_format_found*](../checks/column/patterns/invalid-uuid-format-found.md) | This check validates common UUID and GUID formats and finds any values that are not in the correct format. | *46f6c2f8-038f-4f36-9b87-fbf3fcd341cc* | +| [*invalid_uuid_format_percent*](../checks/column/patterns/invalid-uuid-format-percent.md) | This check validates common UUID and GUID formats and measures the percentage of invalid values. | *46f6c2f8-038f-4f36-9b87-fbf3fcd341cc* | +| [*invalid_ip4_address_format_found*](../checks/column/patterns/invalid-ip4-address-format-found.md) | This check validates the format of IP4 addresses and detects invalid values. | *123.45.67.89* | +| [*invalid_ip6_address_format_found*](../checks/column/patterns/invalid-ip6-address-format-found.md) | This check validates the format of IP6 addresses and detects invalid values. | *66b2:454b:2638:9d9e:b0c1:1c3e:a39c:e83a* | ## Detecting invalid emails The [*invalid_email_format_found*](../checks/column/patterns/invalid-email-format-found.md) check detects invalid emails that do not match the typical email patterns. -The following column profile summary shows a sample email column that contains one invalid email. +The following column profile summary shows a sample email column that contains invalid emails. -![Email column profile with an invalid email](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/email-column-profile-min.png){ loading=lazy; width="1200px" } +![Email column profile with an invalid email](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/email-column-profile-min2.png){ loading=lazy; width="1200px" } ### Detecting invalid emails in UI The [*invalid_email_format_found*](../checks/column/patterns/invalid-email-format-found.md) -check is located in the *patterns* category. The parameter **max_count** configures the maximum accepted number of invalid emails. -The check has found one invalid email. +check is located in the `patterns` category. The parameter **max_count** configures the maximum accepted number of invalid emails. +The check has found four invalid emails. -![Invalid email format data quality check in DQOps editor](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/invalid-email-data-quality-check-in-dqops-editor-min.png){ loading=lazy; width="1200px" }} +![Invalid email format data quality check in DQOps editor](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/invalid-email-data-quality-check-in-dqops-editor-min2.png){ loading=lazy; width="1200px" } -### Detecting invalid emails in UI +### Detecting invalid emails error sampling in UI + +To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. +You can view representative examples of data that do not meet the specified data quality criteria by clicking on the +**Error sampling** tab in the results section. + +![Invalid email format data - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/invalid-email-data-error-sampling.png){ loading=lazy; width="1200px" } + +For additional information about error sampling, please refer to [the Data Quality Error Sampling documentation](../dqo-concepts/data-quality-error-sampling.md). + +### Detecting invalid emails in YAML The configuration of the [*invalid_email_format_found*](../checks/column/patterns/invalid-email-format-found.md) check is simple. ``` { .yaml linenums="1" hl_lines="13-15" } @@ -87,22 +97,35 @@ spec: ## Detecting invalid dates The [*text_not_matching_date_pattern_found*](../checks/column/patterns/text-not-matching-date-pattern-found.md) -and [*text_matching_date_pattern_percent*](../checks/column/patterns/text-matching-date-pattern-percent.md) checks +and [*text_not_matching_date_pattern_percent*](../checks/column/patterns/text-not-matching-date-pattern-percent.md) checks use regular expressions to validate if text columns contain valid date strings that could be later converted to a date type. -The following summary of column profiling shows a text column that contains one invalid date. +The following summary of column profiling shows a text column that contains dates that do not match the YYYY-MM-DD format. -![Date column profile with dates as strings in an invalid format](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-date-column-profile-min.png){ loading=lazy; width="1200px" } +![Date column profile with dates as strings in an invalid format](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-date-column-profile2.png){ loading=lazy; width="1200px" } ### Detecting invalid dates in UI -We will use the [*text_matching_date_pattern_percent*](../checks/column/patterns/text-matching-date-pattern-percent.md) +We will use the [*text_not_matching_date_pattern_percent*](../checks/column/patterns/text-not-matching-date-pattern-percent.md) check to detect invalid date patterns and measure the percentage of valid rows. -The tested table has three rows, and only two contain valid dates. We are expecting that the rate of valid values is 66.6%. -![Text in valid date format data quality check in DQOps editor](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/validating-dates-in-text-columns-dqops-data-quality-check-min.png){ loading=lazy; width="1200px" } +DQOps shows this checks when the *Show advanced checks* are enabled with a checkbox at the top of the +[data quality check editor screen](../dqo-concepts/dqops-user-interface-overview.md#check-editor). This check allows to select *date_format* parameter with different formats. + +After executing the check, we can see in the results that 45% of the rows in this column do not match the YYYY-MM-DD date pattern. + +![Text in valid date format data quality check in DQOps editor](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/validating-dates-in-text-columns-dqops-data-quality-check2.png){ loading=lazy; width="1200px" } + +### Detecting invalid dates error sampling in UI + +We can view representative examples of data not matching YYYY-MM-DD date pattern by clicking on the +**Error sampling** tab in the results section. + +![Text in valid date format - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/validating-dates-in-text-columns-error-sampling.png){ loading=lazy; width="1200px" }} + +For additional information about error sampling, please refer to [the Data Quality Error Sampling documentation](../dqo-concepts/data-quality-error-sampling.md). ### Detecting invalid dates in YAML -The configuration of the [*text_matching_date_pattern_percent*](../checks/column/patterns/text-matching-date-pattern-percent.md) +The configuration of the [*text_not_matching_date_pattern_percent*](../checks/column/patterns/text-not-matching-date-pattern-percent.md) check uses an additional parameter, one of the supported date formats. The supported date formats are: @@ -115,7 +138,7 @@ The supported date formats are: | `YYYY/MM/DD` | *2024/02/10* | | `Month D, YYYY ` | *February 10, 2024* | -The following code sample shows a configured [*text_matching_date_pattern_percent*](../checks/column/patterns/text-matching-date-pattern-percent.md) +The following code sample shows a configured [*text_not_matching_date_pattern_percent*](../checks/column/patterns/text-not-matching-date-pattern-percent.md) check in a DQOps YAML file. ``` { .yaml linenums="1" hl_lines="13-17" } @@ -131,17 +154,17 @@ spec: monitoring_checks: daily: patterns: - daily_text_matching_date_pattern_percent: + daily_text_not_matching_date_pattern_percent: parameters: date_format: YYYY-MM-DD error: - min_percent: 100.0 + max_percent: 0.0 ``` ## Validating custom regular expressions If DQOps does not have a built-in pattern, we can use the [*text_not_matching_regex_found*](../checks/column/patterns/text-not-matching-regex-found.md) -or [*texts_matching_regex_percent*](../checks/column/patterns/texts-matching-regex-percent.md) +or [*texts_not_matching_regex_percent*](../checks/column/patterns/texts-not-matching-regex-percent.md) checks with a custom regular expression pattern. !!! tip "Creating custom pattern data quality checks" @@ -153,16 +176,19 @@ checks with a custom regular expression pattern. that show how to validate a DUNS number with a custom check. ### Validating regular expressions in UI -The [*texts_matching_regex_percent*](../checks/column/patterns/texts-matching-regex-percent.md) -check is configured by setting the regular expression pattern and a minimum accepted percentage of valid values. +The [*texts_not_matching_regex_percent*](../checks/column/patterns/texts-not-matching-regex-percent.md) +check is configured by setting the regular expression pattern and a maximum accepted percentage of invalid values. -We will validate the text column that should contain dates but using a custom regular expression. +We will validate the text column that should contain dates in YYYY-MM-DDD format but using a custom regular expression. The regular expression will be `^[0-9]{4}-[0-9]{2}-[0-9]{2}$`. -![Validate date text with a custom regular expression using DQOps check editor](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/regex-match-data-quality-check-validating-date-string-in-dqops-min.png){ loading=lazy; width="1200px" } +![Validate date text with a custom regular expression using DQOps check editor](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/regex-match-data-quality-check-validating-date-string-in-dqops-min2.png){ loading=lazy; width="1200px" } + +We can view representative examples of data not matching the regular expression by clicking on the +**Error sampling** tab in the results section. ### Validating regular expressions in YAML -The configuration of the [*texts_matching_regex_percent*](../checks/column/patterns/texts-matching-regex-percent.md) +The configuration of the [*texts_not_matching_regex_percent*](../checks/column/patterns/texts-not-matching-regex-percent.md) check in YAML is simple. The regular expression can be wrapped in double quotes to avoid issues when the YAML file is parsed by DQOps. @@ -180,21 +206,21 @@ spec: monitoring_checks: daily: patterns: - daily_texts_matching_regex_percent: + daily_texts_not_matching_regex_percent: parameters: regex: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" error: - min_percent: 100.0 + max_percent: 0.0 ``` ## Use cases -| **Name of the example** | **Description** | -|:--------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Percentage of texts matching a date pattern](../examples/data-validity/percentage-of-texts-matching-date-regex.md) | This example shows how to detect that the percentage of texts matching the date format regex in a column does not exceed a set threshold using [text_matching_date_pattern_percent](../checks/column/patterns/text-matching-date-pattern-percent.md) check. | -| [Detect invalid emails](../examples/data-validity/detect-invalid-emails.md) | This example shows how to detect that the number of invalid emails in a column does not exceed the maximum accepted count using [invalid_email_format_found](../checks/column/patterns/invalid-email-format-found.md) check. | -| [Percentage of valid UUID](../examples/data-validity/percentage-of-valid-uuid.md) | This example shows how to detect that th percentage of valid UUID values in a column does not fall below a set threshold using [valid_uuid_format_percent](../checks/column/patterns/valid-uuid-format-percent.md) check. | -| [Detect invalid IP4 address](../examples/data-validity/detect-invalid-ip4-addresses.md) | This example shows how to detect that the number of invalid IP4 address in a column does not exceed a set threshold using [invalid_ip4_address_format_found](../checks/column/patterns/invalid-ip4-address-format-found.md) check. | +| **Name of the example** | **Description** | +|:------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Percentage of texts not matching a date pattern](../examples/data-validity/percentage-of-texts-matching-date-regex.md) | This example shows how to detect that the percentage of texts not matching the date format regex in a column does not exceed a set threshold using [text_not_matching_date_pattern_percent](../checks/column/patterns/text-not-matching-date-pattern-percent.md) check. | +| [Detect invalid emails](../examples/data-validity/detect-invalid-emails.md) | This example shows how to detect that the number of invalid emails in a column does not exceed the maximum accepted count using [invalid_email_format_found](../checks/column/patterns/invalid-email-format-found.md) check. | +| [Percentage of invalid UUID](../examples/data-validity/percentage-of-valid-uuid.md) | This example shows how to detect that th percentage of valid UUID values in a column does not fall below a set threshold using [invalid_uuid_format_percent](../checks/column/patterns/invalid-uuid-format-percent.md) check. | +| [Detect invalid IP4 address](../examples/data-validity/detect-invalid-ip4-addresses.md) | This example shows how to detect that the number of invalid IP4 address in a column does not exceed a set threshold using [invalid_ip4_address_format_found](../checks/column/patterns/invalid-ip4-address-format-found.md) check. | ## List of patterns checks at a column level | Data quality check name | Friendly name | Data quality dimension | Description | Standard check | diff --git a/docs/categories-of-data-quality-checks/how-to-detect-blank-and-whitespace-values.md b/docs/categories-of-data-quality-checks/how-to-detect-blank-and-whitespace-values.md index 4d0dd92aad..31fd47a6ac 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-blank-and-whitespace-values.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-blank-and-whitespace-values.md @@ -185,7 +185,7 @@ The results in DQOps show that 75% of non-null values (three out of four) contai To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Detect texts surrounded by whitespace characters using DQOps data quality check - error samples](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/detect-text-surrounded-with-whitespace-data-quality-check-in-dqops-min-error-samples.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-bool-fields.md b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-bool-fields.md index 3018fa44e4..5bf1c8c371 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-bool-fields.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-bool-fields.md @@ -38,7 +38,7 @@ Both bool checks accept two optional rule parameters, which set the valid percen To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. For additional information about error sampling, please refer to [the Data Quality Error Sampling documentation](../dqo-concepts/data-quality-error-sampling.md). ### Enable bool checks in YAML diff --git a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-numeric-fields.md b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-numeric-fields.md index 96ddb709b0..0d3549f5c8 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-numeric-fields.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-numeric-fields.md @@ -128,7 +128,7 @@ spec: To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. Let's modify the max_value parameter to 9. Running the checks resulted in warning. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Detect numeric values above a maximum accepted value using a data quality check - error samples](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/detect-out-of-range-numeric-values-above-maximum-value-check-error-samples.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-text-fields.md b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-text-fields.md index 72163b9b37..3ae0564d2b 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-text-fields.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-text-fields.md @@ -13,11 +13,11 @@ The statistics around text values are less sophisticated than calculating metric but observing the length of strings can still reveal many data quality issues. ### Issues with too short texts -Texts shorter than a reasonable minimum are a possible case of potential data corruption. -We can't expect a phone number to be only two digits. -Or an email that has just two letters must be wrong. It is not even enough to include the domain name. +Texts that are shorter than a reasonable minimum may indicate potential data corruption. For example, a phone number +should not be only two digits, and an email address with just two letters is likely to be incorrect +as it does not include the domain name. -Too short texts are a result of: +Too short texts can be a result of: - Someone accidentally truncated the text manually. @@ -34,11 +34,11 @@ Too short texts are a result of: To detect these types of possible data corruption, we can choose a reasonable minimum text length that should fit the smallest valid value, such as a phone number. -Truncated texts that are too short lead to the completeness and uniqueness issues. -If more identifiers are also truncated, we will have duplicate data. +Short, truncated texts can lead to issues with completeness and uniqueness. +If additional identifiers are also truncated, it can result in duplicate data. ### Issues with too long texts -Texts that are longer than expected are caused by other problems. +Texts that are longer than expected can be caused by other problems. - The data is corrupted. @@ -86,30 +86,33 @@ They are very similar to each other, but they can still detect different types o - The [*text_length_in_range_percent*](../checks/column/text/text-length-in-range-percent.md) check measures the percentage of valid texts whose length is within a minimum and maximum accepted length. +- The [*min_word_count*](../checks/column/text/min-word-count.md) check verifies that the minimum word count of the text column is in the range. + +- The [*max_word_count*](../checks/column/text/max-word-count.md) check verifies that the maximum word count of the text is in the range. + ### Profiling the text length DQOps shows the text length statistics on the column's profile screen. The values are: - The **Text min length** shows the length of the shortest text in the column. - - The **Text max length** shows the length of the longest text in the column. - - The **Text mean length** shows the average text length. +- The **Text min word count** shows the minimal number of words in the column. +- The **Text max word count** shows the maximal number of words in the column. -![Data profiling a text column length in DQOps](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/data-profiling-text-column-length-in-dqops-min.png){ loading=lazy; width="1200px" } +![Data profiling a text column length in DQOps](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/data-profiling-text-column-length-in-dqops-min2.png){ loading=lazy; width="1200px" } ## Minimum text length The [*text_min_length*](../checks/column/text/text-min-length.md) has two optional rule parameters. - The **from** parameter configures a minimum text length bottom range. - - The **to** parameter configures a minimum text length upper range. ### Verifying minimum text length in UI The following screenshot shows configuring the [*text_min_length*](../checks/column/text/text-min-length.md) check in the [DQOps data quality check editor](../dqo-concepts/dqops-user-interface-overview.md#check-editor). -![Configuring minimum text length in range data quality check in DQOps](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/min-text-length-in-range-data-quality-check-in-dqops-min.png){ loading=lazy; width="1200px" } +![Configuring minimum text length in range data quality check in DQOps](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/min-text-length-in-range-data-quality-check-in-dqops-min2.png){ loading=lazy; width="1200px" } ### Verifying minimum text length in YAML The configuration of the [*text_min_length*](../checks/column/text/text-min-length.md) check in YAML is simple. @@ -141,10 +144,8 @@ Instead, it is effortless to customize the built-in checks by combining the sens - [*column/text/text_min_length*](../reference/sensors/column/text-column-sensors.md#text-min-length) sensor that finds the length of the shortest text, - - [*column/text/text_max_length*](../reference/sensors/column/text-column-sensors.md#text-max-length) sensor that finds the length of the longest text, - - [*column/text/text_mean_length*](../reference/sensors/column/text-column-sensors.md#text-mean-length) sensor that calculates the average length. @@ -152,14 +153,13 @@ And one of the anomaly or change detection rules: - [*percentile/anomaly_stationary_percentile_moving_average*](../reference/rules/Percentile.md#anomaly-stationary-percentile-moving-average) that finds anomalies in a 90 days time window, - - [*change/change_percent_1_day*](../reference/rules/Change.md#change-percent-1-day) that detects an increase or a decrease in the captured value (such as a maximum text length). The following screenshot shows the configuration of a custom data quality check that detects changes to the minimum or maximum text length. -![Creating custom data quality check that detects anomalies in the maximum text length](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/creating-custom-text-maximum-length-anomaly-detection-check-dqops-min.png){ loading=lazy; width="1200px" } +![Creating custom data quality check that detects anomalies in the maximum text length](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/creating-custom-text-maximum-length-anomaly-detection-check-dqops-min2.png){ loading=lazy; width="1200px" } ## Use cases | **Name of the example** | **Description** | diff --git a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-with-custom-sql.md b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-with-custom-sql.md index c97fdee67c..b0a67870d3 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-with-custom-sql.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-data-quality-issues-with-custom-sql.md @@ -170,7 +170,7 @@ daily monitoring data quality check. To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Compare columns in a data quality check - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/compare-column-values-data-quality-check-in-editor-error-sampling.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-data-referential-integrity-issues.md b/docs/categories-of-data-quality-checks/how-to-detect-data-referential-integrity-issues.md index d8c903a417..eb4cb7b4a0 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-data-referential-integrity-issues.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-data-referential-integrity-issues.md @@ -154,7 +154,7 @@ Our target of 100% rows containing valid country codes was missed by 0.0087%. To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Detecting missing keys using a lookup data quality check in DQOps - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/foreign-key-lookup-check-configured-in-check-editor-min-error-sampling.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-data-type-changes.md b/docs/categories-of-data-quality-checks/how-to-detect-data-type-changes.md index 4700ef4524..70f0284d59 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-data-type-changes.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-data-type-changes.md @@ -109,7 +109,7 @@ is easy to activate. The parameter of the rule is the expected data type. To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Configure data type detection check in UI - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/assert-data-type-in-text-column-street-number-check-error-sampling.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md b/docs/categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md index f3ee532d1f..9761468c49 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md @@ -199,7 +199,7 @@ takes one parameter **max_count**, which is the maximum accepted number of dupli To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for uniqueness checks. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Duplicate count data quality check in DQOps - error samples](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/duplicate-count-check-in-dqops-min-error-samples2.png){ loading=lazy; width="1200px" } @@ -280,47 +280,53 @@ The records should be unique for each combination of the *edition*, *report_type | 2021 | 2021 Health Disparities | Able-Bodied | District of Columbia | American Indian/Alaska Native | 62.0 | 49.0 | 75.0 | U.S. Census Bureau, American Community Survey PUMS | 2015-2019 | ### Configuring multi-column duplicate checks -The uniqueness checks in DQOps operate on single columns. To detect duplicates in a combination of columns, -we have to define a [calculated column](../dqo-concepts/configuring-table-metadata.md#calculated-columns) -derived as a concatenation of all columns that should be unique when combined. +A table-level uniqueness check detects duplicates in a combination of columns. +When all columns are used, it will identify duplicate records in a table with the complete rows. -The SQL expression that concatenates the values will use a `||` concatenation operator, as shown below. -DQOps replaces the `{alias}.` token with the alias of the table. +### Detect multi-column duplicates in UI -`{alias}.edition || {alias}.report_type || {alias}.measure_name || {alias}.state_name || {alias}.subpopulation` +The duplicate records check is available in count and percentage versions. -### Detect multi-column duplicates in UI -We have to add a virtual column to the monitored table. The column will be calculated using the SQL expression shown above. +The [*duplicate_records_count*](../checks/table/uniqueness/duplicate-record-count.md) check counts the distinct records based on the selected columns in columns parameter. + +The [*duplicate_records_percent*](../checks/table/uniqueness/duplicate-record-percent.md) check compares the count of distinct records to the count of all records excluding that containing only nulls based on the selected columns in columns parameter. + + +The multi-column uniqueness checks are available under the table in the tree on the left, in the **Data quality checks editor** tab. + +![Table uniqueness](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-uniqueness.png){ loading=lazy; } -![Adding calculated column for concatenating values for multi-column duplicate detection in DQOps](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/adding-calculated-column-concatenated-unique-values-in-dqops-min.png){ loading=lazy; width="1200px" } +To select the columns used for the duplicate verification, click the **columns** editor. -After adding a calculated column, -DQOps will show it in the metadata tree. We can now configure the [*duplicate_count*](../checks/column/uniqueness/duplicate-count.md) -data quality check. +![Table uniqueness column selection](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-uniqueness-column-selection.png){ loading=lazy; } + +When columns are not configured or none are selected, then all columns are implicitly used by the uniqueness check. -![Duplicate count detection in DQOps on multiple columns using a data quality check editor](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/duplicate-detection-check-in-dqops-on-multiple-columns-min.png){ loading=lazy; width="1200px" } ### Detect multi-column duplicates in YAML -We must add our calculated column to the list of columns to detect duplicates on multiple other columns. -This column will have an additional configuration, the SQL expression discussed before. +The configuration of the [*duplicate_records_percent*](../checks/table/uniqueness/duplicate-record-percent.md) check in a YAML file is shown below. -``` { .yaml linenums="1" hl_lines="7-8 12-14" } +``` { .yaml linenums="1" hl_lines="8-10" } # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table spec: - columns: - unique_columns: - sql_expression: "{alias}.edition || {alias}.report_type || {alias}.measure_name\ - \ || {alias}.state_name || {alias}.subpopulation" - monitoring_checks: - daily: - uniqueness: - daily_duplicate_count: - error: - max_count: 0 + monitoring_checks: + daily: + uniqueness: + daily_duplicate_record_percent: + parameters: + columns: + - edition + - measure_name + - report_type + - state_name + - subpopulation + error: + max_percent: 0.0 ``` + ## Configuring other uniqueness checks The DQOps data quality check editor shows the remaining uniqueness checks after clicking the **Show advanced checks** checkbox. @@ -360,6 +366,20 @@ The reference section provides YAML code samples that are ready to copy-paste to the parameters reference, and samples of data source specific SQL queries generated by [data quality sensors](../dqo-concepts/definition-of-data-quality-sensors.md) that are used by those checks. +## List of uniqueness checks at a table level +| Data quality check name | Friendly name | Data quality dimension | Description | Standard check | +|-------------------------|---------------|------------------------|-------------|----------------| +|[*duplicate_record_count*](../checks/table/uniqueness/duplicate-record-count.md)|Maximum count of duplicate records|[Uniqueness](../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|This check counts duplicate records values. It raises a data quality issue when the number of duplicates is above a minimum accepted value. The default configuration detects duplicate rows by enforcing that the *min_count* of duplicates is zero.|:material-check-bold:| +|[*duplicate_record_percent*](../checks/table/uniqueness/duplicate-record-percent.md)|Maximum percentage of duplicate records|[Uniqueness](../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|This check measures the percentage of duplicate records values. It raises a data quality issue when the percentage of duplicates is above a minimum accepted value. The default threshold is 0% duplicate values.|:material-check-bold:| + + +**Reference and samples** + +The full list of all data quality checks in this category is located in the [table/uniqueness](../checks/table/uniqueness/index.md) reference. +The reference section provides YAML code samples that are ready to copy-paste to the [*.dqotable.yaml*](../reference/yaml/TableYaml.md) files, +the parameters reference, and samples of data source specific SQL queries generated by [data quality sensors](../dqo-concepts/definition-of-data-quality-sensors.md) +that are used by those checks. + ## What's next - Learn how to [run data quality checks](../dqo-concepts/running-data-quality-checks.md#targeting-a-category-of-checks) filtering by a check category name - Learn how to [configure data quality checks](../dqo-concepts/configuring-data-quality-checks-and-rules.md) and apply alerting rules diff --git a/docs/categories-of-data-quality-checks/how-to-detect-empty-or-incomplete-columns-with-nulls.md b/docs/categories-of-data-quality-checks/how-to-detect-empty-or-incomplete-columns-with-nulls.md index 7ef49c0afc..015d0336c4 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-empty-or-incomplete-columns-with-nulls.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-empty-or-incomplete-columns-with-nulls.md @@ -66,7 +66,7 @@ The default value of the *max_count* parameter is 0, which asserts that no null To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Detect incomplete columns with some null values using a data quality check - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/incomplete-column-with-some-nulls-check-in-editor-error-sampling.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-invalid-dates.md b/docs/categories-of-data-quality-checks/how-to-detect-invalid-dates.md index 1d677f846b..b23b373ada 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-invalid-dates.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-invalid-dates.md @@ -41,7 +41,7 @@ The *max_percent* parameter controls the maximum accepted percentage of invalid To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Date in the future invalid date - error samples](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/date-in-future-found-check-error-samples.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-pii-values-and-sensitive-data.md b/docs/categories-of-data-quality-checks/how-to-detect-pii-values-and-sensitive-data.md index bb6c67a1ff..6f5aa8366f 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-pii-values-and-sensitive-data.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-pii-values-and-sensitive-data.md @@ -93,7 +93,7 @@ To display non-standard checks, such as *contains_usa_zipcode_percent* select th The following example shows the result of detecting phone numbers, emails, and zip codes in the **complaint_description** column. DQOps did not detect sensitive data inside any value stored in the column. -![Data quality check editor with PII checks enabled and valid results](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/pii-data-quality-checks-valid-results-min2.png){ loading=lazy; width="1200px" } +![Data quality check editor with PII checks enabled and correct results](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/pii-data-quality-checks-valid-results-min2.png){ loading=lazy; width="1200px" } The next example shows the result of running the same checks on the **incident_address** column. We can see that this public dataset contains a few phone numbers and emails. @@ -104,7 +104,7 @@ We can see that this public dataset contains a few phone numbers and emails. To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for PII check. You can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Data quality checks editor with PII checks that found sensitive data - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/pii-check-detecting-emails-and-phones-found-error-sampling.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-detect-table-schema-changes.md b/docs/categories-of-data-quality-checks/how-to-detect-table-schema-changes.md index 90610a6ac1..cdf733b494 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-table-schema-changes.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-table-schema-changes.md @@ -65,7 +65,7 @@ The change detection checks should be configured as [daily monitoring checks](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md#daily-monitoring-checks) to detect day-to-day changes. -![Enabled data quality checks for table schema drift detection](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-schema-drift-monitoring-enabled-without-changes-min.png){ loading=lazy; width="1200px" } +![Enabled data quality checks for table schema drift detection](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-schema-drift-monitoring-enabled-without-changes-min2.png){ loading=lazy; width="1200px" } ### Example of a schema change In the meantime, a new column was added to the table using an SQL statement shown below. @@ -77,7 +77,7 @@ ADD COLUMN int64_field_31 INTEGER; The following screenshot shows the data quality check editor on the next day. -![Data quality checks that detected table schema changes](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-schema-drift-monitoring-detected-changes-min.png){ loading=lazy; width="1200px" } +![Data quality checks that detected table schema changes](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-schema-drift-monitoring-detected-changes-min2.png){ loading=lazy; width="1200px" } DQOps detected new data quality issues for all types of schema changes because adding a new column is detected by all table-level checks. You can also review the [example of detecting table schema changes](../examples/schema/detect-table-schema-changes.md), @@ -96,15 +96,15 @@ spec: schema: daily_column_count: error: - expected_value: 31 + expected_value: 32 daily_column_count_changed: - error: {} + warning: {} daily_column_list_changed: - error: {} + warning: {} daily_column_list_or_order_changed: - error: {} + warning: {} daily_column_types_changed: - error: {} + warning: {} ``` @@ -122,7 +122,7 @@ DQOps supports the following schema change detection checks configured on a colu The column-level schema change detection checks are configured using the [data quality check editor](../dqo-concepts/dqops-user-interface-overview.md#check-editor) on a column level. -![Configuring column schema change detection data quality checks](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/column-schema-checks-enabled-in-editor-min.png){ loading=lazy; width="1200px" } +![Configuring column schema change detection data quality checks](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/column-schema-checks-enabled-in-editor-min2.png){ loading=lazy; width="1200px" } ### Configuring column-level checks in YAML diff --git a/docs/categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md b/docs/categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md index 6b8f42db8a..0a0de95f73 100644 --- a/docs/categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md +++ b/docs/categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md @@ -157,11 +157,11 @@ You have to configure one or both of the column names: ### Configuring timestamp columns from UI To configure the event and/or ingestion timestamp columns: -1. Go to the **Data Sources** section. +1. Go to the **Data sources** section. 2. Select the table of interest from the tree view. -3. Select the **Data and Time Columns** tab and select a column from the drop-down list in the "Event timestamp column name +3. Select the **Data and time columns** tab and select a column from the drop-down list in the "Event timestamp column name for timeliness checks" and/or "Ingestion timestamp column name for timeliness checks" input fields. 4. Click the Save button in the upper right corner. diff --git a/docs/categories-of-data-quality-checks/how-to-reconcile-data-and-detect-differences.md b/docs/categories-of-data-quality-checks/how-to-reconcile-data-and-detect-differences.md index 7394515fe9..6156f3aff0 100644 --- a/docs/categories-of-data-quality-checks/how-to-reconcile-data-and-detect-differences.md +++ b/docs/categories-of-data-quality-checks/how-to-reconcile-data-and-detect-differences.md @@ -149,7 +149,7 @@ using a background color of each [data quality issue severity levels](../dqo-co ### Tables match When all metrics match, DQOps shows the cells using a green background. -![Table reconciliation valid result tables match](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/data-reconciliation-tables-match-result-min.png){ loading=lazy; width="1200px" } +![Table reconciliation correct result tables match](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/data-reconciliation-tables-match-result-min.png){ loading=lazy; width="1200px" } ### Discrepancies detected We are introducing a discrepancy on the rows whose city value is MANOR. diff --git a/docs/categories-of-data-quality-checks/how-to-table-availability-issues-and-downtimes.md b/docs/categories-of-data-quality-checks/how-to-table-availability-issues-and-downtimes.md index da39347fd9..b600ce941a 100644 --- a/docs/categories-of-data-quality-checks/how-to-table-availability-issues-and-downtimes.md +++ b/docs/categories-of-data-quality-checks/how-to-table-availability-issues-and-downtimes.md @@ -12,15 +12,10 @@ We have a table availability issue when even a simple query cannot be run on a t There are many reasons for table availability issues. - The database server is down. - - Networking issues. - - Firewall issues. - - Invalid permissions to the table. - - The table is physically corrupted on the disk. - - Credentials expired or were changed. @@ -65,17 +60,13 @@ spec: daily_table_availability: warning: max_failures: 0 - error: - max_failures: 5 - fatal: - max_failures: 10 columns: {} ``` ### User interface The table availability monitoring can be also enabled in the [check editor](../dqo-concepts/dqops-user-interface-overview.md#check-editor). -![Table availability detection configuration in DQOps](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-availability-check-detection-editor-min.png){ loading=lazy; width="1200px" } +![Table availability detection configuration in DQOps](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/table-availability-check-detection-editor-min2.png){ loading=lazy; width="1200px" } ## Monitoring table availability on dashboards DQOps provides dedicated data quality dashboards for monitoring table availability issues. diff --git a/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md b/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md index e521fcee70..db887d60ec 100644 --- a/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md +++ b/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md @@ -78,7 +78,7 @@ spec: To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. Let's remove the CALDWELL value from the **expected_values** and run the check again. Now the check failed and you can view representative examples of data that do not meet the specified data quality criteria by clicking on the -**Error Sample** tab in the results section. +**Error sampling** tab in the results section. ![Text found in set percent - error samples](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-found-in-set-percent-error-samples.png){ loading=lazy; width="1200px" } diff --git a/docs/categories-of-data-quality-checks/how-to-verify-text-values-are-parsable.md b/docs/categories-of-data-quality-checks/how-to-verify-text-values-are-parsable.md index 98a73795df..6c8b1ed8e4 100644 --- a/docs/categories-of-data-quality-checks/how-to-verify-text-values-are-parsable.md +++ b/docs/categories-of-data-quality-checks/how-to-verify-text-values-are-parsable.md @@ -71,7 +71,7 @@ The following example shows how to validate identifier values in a raw table sto The identifier column is *product_id*, which should contain only integer values. The data profiling screen reveals that the column contains a non-numeric value "3Product". -![Text column with integers and one invalid text that cannot be converted to a number](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/integer-with-invalid-values-column-profile-min.png){ loading=lazy; width="1200px" } +![Text column with integers and one invalid text that cannot be converted to a number](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/integer-with-invalid-values-column-profile-min2.png){ loading=lazy; width="1200px" } ### Testing valid integers in UI The [*text_parsable_to_integer_percent*](../checks/column/conversions/text-parsable-to-integer-percent.md) check @@ -80,7 +80,17 @@ It measures the percentage of valid values. The data quality check is configured by setting the **min_percent** parameter, which is the minimum percentage of convertible values. -![Data conversion data quality check to parse text values as integer values](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-value-convertible-to-integer-data-quality-check-min.png){ loading=lazy; width="1200px" } +![Data conversion data quality check to parse text values as integer values](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-value-convertible-to-integer-data-quality-check-min2.png){ loading=lazy; width="1200px" } + +### Valid integers error sampling in UI + +To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling for this check. +You can view representative examples of data that do not meet the specified data quality criteria by clicking on the +**Error sampling** tab in the results section. + +![Data conversion data quality check to parse text values as integer values - error sampling](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-value-convertible-to-integer-data-quality-check-error-sampling.png){ loading=lazy; width="1200px" } + +For additional information about error sampling, please refer to [the Data Quality Error Sampling documentation](../dqo-concepts/data-quality-error-sampling.md). ### Testing valid integers in YAML The [*text_parsable_to_integer_percent*](../checks/column/conversions/text-parsable-to-integer-percent.md) data quality check is easy to configure. diff --git a/docs/checks/column/datetime/text-match-date-format-percent.md b/docs/checks/column/datetime/text-match-date-format-percent.md index 7dc1f4e2c3..d3c02ffec0 100644 --- a/docs/checks/column/datetime/text-match-date-format-percent.md +++ b/docs/checks/column/datetime/text-match-date-format-percent.md @@ -271,7 +271,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -868,7 +868,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -1573,7 +1573,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -2171,7 +2171,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -2876,7 +2876,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -3474,7 +3474,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -4201,7 +4201,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -4859,7 +4859,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -5608,7 +5608,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END @@ -6266,7 +6266,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END diff --git a/docs/checks/column/numeric/invalid-latitude.md b/docs/checks/column/numeric/invalid-latitude.md index 6d1049d0f5..05753d8d1b 100644 --- a/docs/checks/column/numeric/invalid-latitude.md +++ b/docs/checks/column/numeric/invalid-latitude.md @@ -139,8 +139,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -156,8 +156,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `your-google-project-id`.``.`` AS analyzed_table @@ -171,8 +171,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -188,8 +188,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -203,8 +203,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -220,8 +220,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM AS analyzed_table @@ -235,8 +235,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -252,8 +252,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `` AS analyzed_table @@ -267,8 +267,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -290,8 +290,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM( @@ -309,8 +309,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -326,8 +326,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_postgresql_database".""."" AS analyzed_table @@ -341,8 +341,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -365,8 +365,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -384,8 +384,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -401,8 +401,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_redshift_database".""."" AS analyzed_table @@ -416,8 +416,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -433,8 +433,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_snowflake_database".""."" AS analyzed_table @@ -448,8 +448,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -465,8 +465,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -480,8 +480,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -497,8 +497,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -512,8 +512,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -536,8 +536,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -602,8 +602,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -618,8 +618,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -636,8 +636,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -652,8 +652,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -670,8 +670,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -686,8 +686,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -704,8 +704,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -720,8 +720,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -738,8 +738,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -760,8 +760,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -787,8 +787,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -803,8 +803,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -821,8 +821,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -844,8 +844,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -871,8 +871,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -887,8 +887,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -905,8 +905,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -921,8 +921,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -939,8 +939,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -955,8 +955,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -973,8 +973,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -989,8 +989,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -1011,8 +1011,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1034,8 +1034,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -1185,8 +1185,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1202,8 +1202,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `your-google-project-id`.``.`` AS analyzed_table @@ -1217,8 +1217,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1234,8 +1234,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -1249,8 +1249,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1266,8 +1266,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM AS analyzed_table @@ -1281,8 +1281,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1298,8 +1298,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `` AS analyzed_table @@ -1313,8 +1313,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1336,8 +1336,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM( @@ -1355,8 +1355,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1372,8 +1372,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_postgresql_database".""."" AS analyzed_table @@ -1387,8 +1387,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1411,8 +1411,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -1430,8 +1430,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1447,8 +1447,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_redshift_database".""."" AS analyzed_table @@ -1462,8 +1462,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1479,8 +1479,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_snowflake_database".""."" AS analyzed_table @@ -1494,8 +1494,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1511,8 +1511,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -1526,8 +1526,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1543,8 +1543,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -1558,8 +1558,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1582,8 +1582,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -1649,8 +1649,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1665,8 +1665,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -1683,8 +1683,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1699,8 +1699,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -1717,8 +1717,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1733,8 +1733,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1751,8 +1751,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1767,8 +1767,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -1785,8 +1785,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1807,8 +1807,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -1834,8 +1834,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1850,8 +1850,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1868,8 +1868,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1891,8 +1891,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -1918,8 +1918,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1934,8 +1934,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1952,8 +1952,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1968,8 +1968,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1986,8 +1986,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2002,8 +2002,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2020,8 +2020,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2036,8 +2036,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -2058,8 +2058,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2081,8 +2081,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -2232,8 +2232,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2249,8 +2249,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `your-google-project-id`.``.`` AS analyzed_table @@ -2264,8 +2264,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2281,8 +2281,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -2296,8 +2296,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2313,8 +2313,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM AS analyzed_table @@ -2328,8 +2328,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2345,8 +2345,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `` AS analyzed_table @@ -2360,8 +2360,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2383,8 +2383,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM( @@ -2402,8 +2402,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2419,8 +2419,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_postgresql_database".""."" AS analyzed_table @@ -2434,8 +2434,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2458,8 +2458,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -2477,8 +2477,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2494,8 +2494,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_redshift_database".""."" AS analyzed_table @@ -2509,8 +2509,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2526,8 +2526,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_snowflake_database".""."" AS analyzed_table @@ -2541,8 +2541,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2558,8 +2558,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -2573,8 +2573,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2590,8 +2590,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -2605,8 +2605,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2629,8 +2629,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -2696,8 +2696,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2712,8 +2712,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2730,8 +2730,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2746,8 +2746,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2764,8 +2764,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2780,8 +2780,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -2798,8 +2798,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2814,8 +2814,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2832,8 +2832,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2854,8 +2854,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -2881,8 +2881,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2897,8 +2897,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -2915,8 +2915,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2938,8 +2938,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -2965,8 +2965,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2981,8 +2981,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -2999,8 +2999,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3015,8 +3015,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -3033,8 +3033,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3049,8 +3049,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3067,8 +3067,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3083,8 +3083,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -3105,8 +3105,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3128,8 +3128,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -3289,8 +3289,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3306,8 +3306,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.`date_column` AS DATE) AS time_period, @@ -3325,8 +3325,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3342,8 +3342,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.`date_column` AS DATE) AS time_period, @@ -3361,8 +3361,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3378,8 +3378,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3397,8 +3397,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3414,8 +3414,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, @@ -3433,8 +3433,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3456,8 +3456,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -3481,8 +3481,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3498,8 +3498,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3517,8 +3517,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3541,8 +3541,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -3566,8 +3566,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3583,8 +3583,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3602,8 +3602,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3619,8 +3619,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3638,8 +3638,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3655,8 +3655,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.`date_column` AS DATE) AS time_period, @@ -3674,8 +3674,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3691,8 +3691,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.[date_column] AS date) AS time_period, @@ -3712,8 +3712,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3736,8 +3736,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -3819,8 +3819,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3835,8 +3835,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3855,8 +3855,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3871,8 +3871,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3891,8 +3891,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3907,8 +3907,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -3927,8 +3927,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3943,8 +3943,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3963,8 +3963,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3985,8 +3985,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -4016,8 +4016,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4032,8 +4032,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -4052,8 +4052,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4075,8 +4075,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -4106,8 +4106,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4122,8 +4122,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -4142,8 +4142,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4158,8 +4158,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -4178,8 +4178,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4194,8 +4194,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -4214,8 +4214,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4230,8 +4230,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -4252,8 +4252,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4275,8 +4275,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -4440,8 +4440,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4457,8 +4457,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, @@ -4476,8 +4476,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4493,8 +4493,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, @@ -4512,8 +4512,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4529,8 +4529,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4548,8 +4548,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4565,8 +4565,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, @@ -4584,8 +4584,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4607,8 +4607,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -4632,8 +4632,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4649,8 +4649,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4668,8 +4668,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4692,8 +4692,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -4717,8 +4717,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4734,8 +4734,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4753,8 +4753,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4770,8 +4770,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4789,8 +4789,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4806,8 +4806,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, @@ -4825,8 +4825,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4842,8 +4842,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, @@ -4863,8 +4863,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4887,8 +4887,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -4970,8 +4970,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4986,8 +4986,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5006,8 +5006,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5022,8 +5022,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5042,8 +5042,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5058,8 +5058,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5078,8 +5078,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5094,8 +5094,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5114,8 +5114,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -5136,8 +5136,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -5167,8 +5167,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5183,8 +5183,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5203,8 +5203,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -5226,8 +5226,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -5257,8 +5257,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5273,8 +5273,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5293,8 +5293,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5309,8 +5309,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5329,8 +5329,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5345,8 +5345,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -90.0 AND analyzed_table.`target_column` <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -90.0 OR analyzed_table.`target_column` > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5365,8 +5365,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5381,8 +5381,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -90.0 AND analyzed_table.[target_column] <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -90.0 OR analyzed_table.[target_column] > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -5403,8 +5403,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -5426,8 +5426,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -90.0 AND analyzed_table."target_column" <= 90.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -90.0 OR analyzed_table."target_column" > 90.0 THEN 1 + ELSE 0 END ) AS actual_value, diff --git a/docs/checks/column/numeric/invalid-longitude.md b/docs/checks/column/numeric/invalid-longitude.md index 94dcceec83..06fb3352fc 100644 --- a/docs/checks/column/numeric/invalid-longitude.md +++ b/docs/checks/column/numeric/invalid-longitude.md @@ -139,8 +139,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -156,8 +156,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `your-google-project-id`.``.`` AS analyzed_table @@ -171,8 +171,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -188,8 +188,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -203,8 +203,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -220,8 +220,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM AS analyzed_table @@ -235,8 +235,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -252,8 +252,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `` AS analyzed_table @@ -267,8 +267,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -290,8 +290,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM( @@ -309,8 +309,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -326,8 +326,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_postgresql_database".""."" AS analyzed_table @@ -341,8 +341,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -365,8 +365,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -384,8 +384,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -401,8 +401,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_redshift_database".""."" AS analyzed_table @@ -416,8 +416,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -433,8 +433,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_snowflake_database".""."" AS analyzed_table @@ -448,8 +448,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -465,8 +465,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -480,8 +480,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -497,8 +497,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -512,8 +512,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -536,8 +536,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -602,8 +602,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -618,8 +618,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -636,8 +636,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -652,8 +652,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -670,8 +670,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -686,8 +686,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -704,8 +704,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -720,8 +720,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -738,8 +738,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -760,8 +760,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -787,8 +787,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -803,8 +803,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -821,8 +821,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -844,8 +844,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -871,8 +871,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -887,8 +887,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -905,8 +905,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -921,8 +921,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -939,8 +939,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -955,8 +955,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -973,8 +973,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -989,8 +989,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -1011,8 +1011,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1034,8 +1034,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -1185,8 +1185,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1202,8 +1202,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `your-google-project-id`.``.`` AS analyzed_table @@ -1217,8 +1217,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1234,8 +1234,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -1249,8 +1249,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1266,8 +1266,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM AS analyzed_table @@ -1281,8 +1281,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1298,8 +1298,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `` AS analyzed_table @@ -1313,8 +1313,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1336,8 +1336,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM( @@ -1355,8 +1355,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1372,8 +1372,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_postgresql_database".""."" AS analyzed_table @@ -1387,8 +1387,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1411,8 +1411,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -1430,8 +1430,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1447,8 +1447,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_redshift_database".""."" AS analyzed_table @@ -1462,8 +1462,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1479,8 +1479,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_snowflake_database".""."" AS analyzed_table @@ -1494,8 +1494,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1511,8 +1511,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -1526,8 +1526,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1543,8 +1543,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -1558,8 +1558,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1582,8 +1582,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -1649,8 +1649,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1665,8 +1665,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -1683,8 +1683,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1699,8 +1699,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -1717,8 +1717,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1733,8 +1733,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1751,8 +1751,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1767,8 +1767,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -1785,8 +1785,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1807,8 +1807,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -1834,8 +1834,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1850,8 +1850,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1868,8 +1868,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -1891,8 +1891,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -1918,8 +1918,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1934,8 +1934,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1952,8 +1952,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1968,8 +1968,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -1986,8 +1986,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2002,8 +2002,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2020,8 +2020,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2036,8 +2036,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -2058,8 +2058,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2081,8 +2081,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -2232,8 +2232,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2249,8 +2249,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `your-google-project-id`.``.`` AS analyzed_table @@ -2264,8 +2264,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2281,8 +2281,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -2296,8 +2296,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2313,8 +2313,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM AS analyzed_table @@ -2328,8 +2328,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2345,8 +2345,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM `` AS analyzed_table @@ -2360,8 +2360,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2383,8 +2383,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM( @@ -2402,8 +2402,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2419,8 +2419,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_postgresql_database".""."" AS analyzed_table @@ -2434,8 +2434,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2458,8 +2458,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -2477,8 +2477,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2494,8 +2494,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_redshift_database".""."" AS analyzed_table @@ -2509,8 +2509,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2526,8 +2526,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM "your_snowflake_database".""."" AS analyzed_table @@ -2541,8 +2541,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2558,8 +2558,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ``.`` AS analyzed_table @@ -2573,8 +2573,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2590,8 +2590,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -2605,8 +2605,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2629,8 +2629,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value FROM ( @@ -2696,8 +2696,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2712,8 +2712,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2730,8 +2730,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2746,8 +2746,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2764,8 +2764,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2780,8 +2780,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -2798,8 +2798,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2814,8 +2814,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -2832,8 +2832,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2854,8 +2854,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -2881,8 +2881,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2897,8 +2897,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -2915,8 +2915,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -2938,8 +2938,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -2965,8 +2965,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2981,8 +2981,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -2999,8 +2999,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3015,8 +3015,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -3033,8 +3033,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3049,8 +3049,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3067,8 +3067,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3083,8 +3083,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -3105,8 +3105,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3128,8 +3128,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -3289,8 +3289,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3306,8 +3306,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.`date_column` AS DATE) AS time_period, @@ -3325,8 +3325,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3342,8 +3342,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.`date_column` AS DATE) AS time_period, @@ -3361,8 +3361,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3378,8 +3378,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3397,8 +3397,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3414,8 +3414,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, @@ -3433,8 +3433,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3456,8 +3456,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -3481,8 +3481,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3498,8 +3498,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3517,8 +3517,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3541,8 +3541,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -3566,8 +3566,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3583,8 +3583,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3602,8 +3602,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3619,8 +3619,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table."date_column" AS date) AS time_period, @@ -3638,8 +3638,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3655,8 +3655,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.`date_column` AS DATE) AS time_period, @@ -3674,8 +3674,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3691,8 +3691,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.[date_column] AS date) AS time_period, @@ -3712,8 +3712,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3736,8 +3736,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -3819,8 +3819,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3835,8 +3835,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3855,8 +3855,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3871,8 +3871,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3891,8 +3891,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3907,8 +3907,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -3927,8 +3927,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3943,8 +3943,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -3963,8 +3963,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -3985,8 +3985,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -4016,8 +4016,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4032,8 +4032,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -4052,8 +4052,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4075,8 +4075,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -4106,8 +4106,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4122,8 +4122,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -4142,8 +4142,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4158,8 +4158,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -4178,8 +4178,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4194,8 +4194,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -4214,8 +4214,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4230,8 +4230,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -4252,8 +4252,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4275,8 +4275,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -4440,8 +4440,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4457,8 +4457,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, @@ -4476,8 +4476,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4493,8 +4493,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, @@ -4512,8 +4512,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4529,8 +4529,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4548,8 +4548,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4565,8 +4565,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, @@ -4584,8 +4584,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4607,8 +4607,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -4632,8 +4632,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4649,8 +4649,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4668,8 +4668,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4692,8 +4692,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -4717,8 +4717,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4734,8 +4734,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4753,8 +4753,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4770,8 +4770,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, @@ -4789,8 +4789,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4806,8 +4806,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, @@ -4825,8 +4825,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4842,8 +4842,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, @@ -4863,8 +4863,8 @@ spec: SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -4887,8 +4887,8 @@ spec: SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, time_period, @@ -4970,8 +4970,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4986,8 +4986,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5006,8 +5006,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5022,8 +5022,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5042,8 +5042,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5058,8 +5058,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5078,8 +5078,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5094,8 +5094,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5114,8 +5114,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -5136,8 +5136,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -5167,8 +5167,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5183,8 +5183,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5203,8 +5203,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -5226,8 +5226,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, @@ -5257,8 +5257,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5273,8 +5273,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5293,8 +5293,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5309,8 +5309,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 AND analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table."country" AS grouping_level_1, @@ -5329,8 +5329,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5345,8 +5345,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.`target_column` >= -180.0 AND analyzed_table.`target_column` <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.`target_column` < -180.0 OR analyzed_table.`target_column` > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.`country` AS grouping_level_1, @@ -5365,8 +5365,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5381,8 +5381,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table.[target_column] >= -180.0 AND analyzed_table.[target_column] <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] < -180.0 OR analyzed_table.[target_column] > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -5403,8 +5403,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -5426,8 +5426,8 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN analyzed_table."target_column" >= -180.0 AND analyzed_table."target_column" <= 180.0 THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" < -180.0 OR analyzed_table."target_column" > 180.0 THEN 1 + ELSE 0 END ) AS actual_value, diff --git a/docs/checks/column/patterns/invalid-email-format-percent.md b/docs/checks/column/patterns/invalid-email-format-percent.md index 5b5594594b..5b3c0d17a4 100644 --- a/docs/checks/column/patterns/invalid-email-format-percent.md +++ b/docs/checks/column/patterns/invalid-email-format-percent.md @@ -141,10 +141,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -163,10 +163,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -226,9 +226,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -248,9 +248,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -267,10 +267,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -289,9 +289,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -308,10 +308,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -336,10 +336,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -360,10 +360,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -382,10 +382,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -402,10 +402,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -432,10 +432,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -457,9 +457,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -478,9 +478,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -497,10 +497,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -519,10 +519,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -539,10 +539,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -561,10 +561,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -581,14 +581,14 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -607,14 +607,14 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value @@ -631,10 +631,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -661,10 +661,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -733,10 +733,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -754,10 +754,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -822,9 +822,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -843,9 +843,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -865,10 +865,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -886,9 +886,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -908,10 +908,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -935,10 +935,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -967,10 +967,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -988,10 +988,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -1011,10 +1011,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -1040,10 +1040,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -1073,9 +1073,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1093,9 +1093,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -1115,10 +1115,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1136,10 +1136,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -1159,10 +1159,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1180,10 +1180,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -1203,14 +1203,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1228,14 +1228,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value, @@ -1259,10 +1259,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -1288,10 +1288,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -1445,10 +1445,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1467,10 +1467,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -1530,9 +1530,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1552,9 +1552,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -1571,10 +1571,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1593,9 +1593,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -1612,10 +1612,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1640,10 +1640,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -1664,10 +1664,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1686,10 +1686,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -1706,10 +1706,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -1736,10 +1736,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -1761,9 +1761,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1782,9 +1782,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -1801,10 +1801,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1823,10 +1823,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -1843,10 +1843,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1865,10 +1865,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -1885,14 +1885,14 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -1911,14 +1911,14 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value @@ -1935,10 +1935,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -1965,10 +1965,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -2038,10 +2038,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2059,10 +2059,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -2127,9 +2127,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2148,9 +2148,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -2170,10 +2170,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2191,9 +2191,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -2213,10 +2213,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2240,10 +2240,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -2272,10 +2272,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2293,10 +2293,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -2316,10 +2316,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -2345,10 +2345,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -2378,9 +2378,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2398,9 +2398,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -2420,10 +2420,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2441,10 +2441,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -2464,10 +2464,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2485,10 +2485,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -2508,14 +2508,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2533,14 +2533,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value, @@ -2564,10 +2564,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -2593,10 +2593,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -2750,10 +2750,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2772,10 +2772,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -2835,9 +2835,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2857,9 +2857,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -2876,10 +2876,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2898,9 +2898,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -2917,10 +2917,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2945,10 +2945,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -2969,10 +2969,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -2991,10 +2991,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -3011,10 +3011,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -3041,10 +3041,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -3066,9 +3066,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3087,9 +3087,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -3106,10 +3106,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3128,10 +3128,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value @@ -3148,10 +3148,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3170,10 +3170,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value @@ -3190,14 +3190,14 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3216,14 +3216,14 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value @@ -3240,10 +3240,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -3270,10 +3270,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -3343,10 +3343,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3364,10 +3364,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -3432,9 +3432,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3453,9 +3453,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -3475,10 +3475,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3496,9 +3496,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -3518,10 +3518,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3545,10 +3545,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -3577,10 +3577,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3598,10 +3598,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -3621,10 +3621,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -3650,10 +3650,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -3683,9 +3683,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3703,9 +3703,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -3725,10 +3725,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3746,10 +3746,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -3769,10 +3769,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3790,10 +3790,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -3813,14 +3813,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -3838,14 +3838,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value, @@ -3869,10 +3869,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -3898,10 +3898,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -4065,10 +4065,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4087,10 +4087,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -4158,9 +4158,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4180,9 +4180,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -4203,10 +4203,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4225,9 +4225,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -4248,10 +4248,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4276,10 +4276,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -4306,10 +4306,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4328,10 +4328,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -4352,10 +4352,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -4382,10 +4382,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -4413,9 +4413,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4434,9 +4434,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -4457,10 +4457,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4479,10 +4479,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -4503,10 +4503,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4525,10 +4525,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -4549,14 +4549,14 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4575,14 +4575,14 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value, @@ -4605,10 +4605,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -4635,10 +4635,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -4724,10 +4724,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4745,10 +4745,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -4817,9 +4817,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4838,9 +4838,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -4862,10 +4862,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4883,9 +4883,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -4907,10 +4907,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4934,10 +4934,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -4970,10 +4970,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -4991,10 +4991,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5016,10 +5016,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -5045,10 +5045,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -5082,9 +5082,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5102,9 +5102,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5126,10 +5126,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5147,10 +5147,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5172,10 +5172,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5193,10 +5193,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -5218,14 +5218,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5243,14 +5243,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value, @@ -5274,10 +5274,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -5303,10 +5303,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -5474,10 +5474,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5496,10 +5496,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -5567,9 +5567,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5589,9 +5589,9 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5612,10 +5612,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5634,9 +5634,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -5657,10 +5657,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5685,10 +5685,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5715,10 +5715,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5737,10 +5737,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5761,10 +5761,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -5791,10 +5791,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -5822,9 +5822,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5843,9 +5843,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5866,10 +5866,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5888,10 +5888,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -5912,10 +5912,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5934,10 +5934,10 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -5958,14 +5958,14 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -5984,14 +5984,14 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value, @@ -6014,10 +6014,10 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -6044,10 +6044,10 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -6133,10 +6133,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6154,10 +6154,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -6226,9 +6226,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6247,9 +6247,9 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -6271,10 +6271,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6292,9 +6292,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^[a-zA-Z0-9.!#$%&*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -6316,10 +6316,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6343,10 +6343,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column" , + WHEN NOT REGEXP_LIKE(analyzed_table."target_column" , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -6379,10 +6379,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6400,10 +6400,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -6425,10 +6425,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -6454,10 +6454,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END @@ -6491,9 +6491,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6511,9 +6511,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -6535,10 +6535,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6556,10 +6556,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table."target_column") END AS actual_value, @@ -6581,10 +6581,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6602,10 +6602,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST(analyzed_table.`target_column` AS STRING), + WHEN NOT REGEXP(CAST(analyzed_table.`target_column` AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT(analyzed_table.`target_column`) END AS actual_value, @@ -6627,14 +6627,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -6652,14 +6652,14 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( analyzed_table.[target_column] LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len(analyzed_table.[target_column]) - len(replace(analyzed_table.[target_column], '@', '')) = 1 -- single use of @ char AND right(analyzed_table.[target_column], len(analyzed_table.[target_column]) - charindex('@', analyzed_table.[target_column])) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left(analyzed_table.[target_column], charindex('@', analyzed_table.[target_column]))) < 64 -- local part length AND analyzed_table.[target_column] not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG(analyzed_table.[target_column]) END AS actual_value, @@ -6683,10 +6683,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -6712,10 +6712,10 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT(analyzed_table."target_column") END diff --git a/docs/checks/column/patterns/invalid-uuid-format-found.md b/docs/checks/column/patterns/invalid-uuid-format-found.md index 544e7ed419..7f375864be 100644 --- a/docs/checks/column/patterns/invalid-uuid-format-found.md +++ b/docs/checks/column/patterns/invalid-uuid-format-found.md @@ -508,9 +508,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -526,9 +527,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -1033,9 +1035,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1050,9 +1053,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -1618,9 +1622,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -1636,9 +1641,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -2144,9 +2150,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2161,9 +2168,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -2729,9 +2737,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2747,9 +2756,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value FROM [your_sql_server_database].[].[] AS analyzed_table @@ -3255,9 +3265,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3272,9 +3283,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -3894,9 +3906,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -3912,9 +3925,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value, CAST(analyzed_table.[date_column] AS date) AS time_period, @@ -4466,9 +4480,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -4483,9 +4498,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, @@ -5109,9 +5125,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5127,9 +5144,10 @@ spec: SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value, DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, @@ -5681,9 +5699,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -5698,9 +5717,10 @@ Expand the *Configure with data grouping* section to see additional examples for SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL - THEN 0 - ELSE 1 + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, analyzed_table.[target_column]) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value, analyzed_table.[country] AS grouping_level_1, diff --git a/docs/checks/column/patterns/invalid-uuid-format-percent.md b/docs/checks/column/patterns/invalid-uuid-format-percent.md index 4d0473efeb..32d1cf0f74 100644 --- a/docs/checks/column/patterns/invalid-uuid-format-percent.md +++ b/docs/checks/column/patterns/invalid-uuid-format-percent.md @@ -568,7 +568,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -589,7 +590,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -1163,7 +1165,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -1183,7 +1186,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -1818,7 +1822,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -1839,7 +1844,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -2414,7 +2420,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2434,7 +2441,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -3069,7 +3077,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3090,7 +3099,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -3665,7 +3675,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3685,7 +3696,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -4374,7 +4386,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -4395,7 +4408,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -5016,7 +5030,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5036,7 +5051,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -5729,7 +5745,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5750,7 +5767,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END @@ -6371,7 +6389,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -6391,7 +6410,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NULL THEN 1 ELSE 0 END diff --git a/docs/checks/column/patterns/text-matching-date-pattern-percent.md b/docs/checks/column/patterns/text-matching-date-pattern-percent.md deleted file mode 100644 index 96bbb80f1c..0000000000 --- a/docs/checks/column/patterns/text-matching-date-pattern-percent.md +++ /dev/null @@ -1,6559 +0,0 @@ ---- -title: text matching date pattern percent data quality checks ---- -# text matching date pattern percent data quality checks - -This check validates the date format of dates stored in text columns. - It measures the percentage of correctly formatted dates and raises a data quality issue when the rate is below a threshold. - - -___ -The **text matching date pattern percent** data quality check has the following variants for each -[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. - - -## profile text matching date pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the date format regular expression in a column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`profile_text_matching_date_pattern_percent`|Minimum percentage of rows containing texts matching an expected date pattern|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_date_patterns_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile text matching date pattern percent data quality check. - -??? example "Managing profile text matching date pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_text_matching_date_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_date_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_date_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_text_matching_date_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_date_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_date_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *profile_text_matching_date_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=profile_text_matching_date_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_text_matching_date_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *profile_text_matching_date_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=profile_text_matching_date_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-17" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - profiling_checks: - patterns: - profile_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 29-34" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - profiling_checks: - patterns: - profile_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily text matching date pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the date format regular expression in a column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_text_matching_date_pattern_percent`|Minimum percentage of rows containing texts matching an expected date pattern|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_date_patterns_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily text matching date pattern percent data quality check. - -??? example "Managing daily text matching date pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_text_matching_date_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_date_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_date_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_text_matching_date_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_date_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_date_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_text_matching_date_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_text_matching_date_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_text_matching_date_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_text_matching_date_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_text_matching_date_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-18" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 30-35" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## monthly text matching date pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the date format regular expression in a column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_text_matching_date_pattern_percent`|Minimum percentage of rows containing texts matching an expected date pattern|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_date_patterns_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly text matching date pattern percent data quality check. - -??? example "Managing monthly text matching date pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_text_matching_date_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_date_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_date_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_text_matching_date_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_date_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_date_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_text_matching_date_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_text_matching_date_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_text_matching_date_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_text_matching_date_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_text_matching_date_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-18" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 30-35" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily partition text matching date pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the date format regular expression in a column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_partition_text_matching_date_pattern_percent`|Minimum percentage of rows containing texts matching an expected date pattern|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_date_patterns_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition text matching date pattern percent data quality check. - -??? example "Managing daily partition text matching date pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_text_matching_date_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_date_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_date_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_text_matching_date_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_date_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_date_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_partition_text_matching_date_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_partition_text_matching_date_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_text_matching_date_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_partition_text_matching_date_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_partition_text_matching_date_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-23" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 40-45" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - -## monthly partition text matching date pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the date format regular expression in a column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_partition_text_matching_date_pattern_percent`|Minimum percentage of rows containing texts matching an expected date pattern|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_date_patterns_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition text matching date pattern percent data quality check. - -??? example "Managing monthly partition text matching date pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_text_matching_date_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_date_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_date_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_text_matching_date_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_date_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_date_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_partition_text_matching_date_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_partition_text_matching_date_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_text_matching_date_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_partition_text_matching_date_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_partition_text_matching_date_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-23" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 40-45" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_text_matching_date_pattern_percent: - parameters: - date_format: YYYY-MM-DD - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_date_patterns_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-date-patterns-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format_regex(parameters.date_format)}}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^([0-9]{4})[-](0[1-9]|1[0-2])[-](0[1-9]|[1][0-9]|[2][0-9]|3[01])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{lib.render_date_format_regex(parameters.date_format)}} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NOT NULL - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - - -## What's next -- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps -- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/column/patterns/text-matching-name-pattern-percent.md b/docs/checks/column/patterns/text-matching-name-pattern-percent.md deleted file mode 100644 index a1de489bed..0000000000 --- a/docs/checks/column/patterns/text-matching-name-pattern-percent.md +++ /dev/null @@ -1,6479 +0,0 @@ ---- -title: text matching name pattern percent data quality checks ---- -# text matching name pattern percent data quality checks - -This check verifies if values stored in a text column contain only letters and are usable as literal identifiers. - It measures the percentage of valid literal identifiers and raises a data quality issue when the rate is below a threshold. - - -___ -The **text matching name pattern percent** data quality check has the following variants for each -[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. - - -## profile text matching name pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the name regular expression does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`profile_text_matching_name_pattern_percent`|Minimum percentage of rows containing texts that are valid names (not numeric)|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_name_pattern_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile text matching name pattern percent data quality check. - -??? example "Managing profile text matching name pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_text_matching_name_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_name_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_name_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_text_matching_name_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_name_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_text_matching_name_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *profile_text_matching_name_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=profile_text_matching_name_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_text_matching_name_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *profile_text_matching_name_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=profile_text_matching_name_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-15" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - profiling_checks: - patterns: - profile_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 27-32" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - profiling_checks: - patterns: - profile_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily text matching name pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the name regular expression does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_text_matching_name_pattern_percent`|Minimum percentage of rows containing texts that are valid names (not numeric)|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_name_pattern_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily text matching name pattern percent data quality check. - -??? example "Managing daily text matching name pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_text_matching_name_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_name_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_name_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_text_matching_name_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_name_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_text_matching_name_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_text_matching_name_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_text_matching_name_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_text_matching_name_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_text_matching_name_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_text_matching_name_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## monthly text matching name pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the name regular expression does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_text_matching_name_pattern_percent`|Minimum percentage of rows containing texts that are valid names (not numeric)|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_name_pattern_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly text matching name pattern percent data quality check. - -??? example "Managing monthly text matching name pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_text_matching_name_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_name_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_name_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_text_matching_name_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_name_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_text_matching_name_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_text_matching_name_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_text_matching_name_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_text_matching_name_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_text_matching_name_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_text_matching_name_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily partition text matching name pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the name regular expression does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_partition_text_matching_name_pattern_percent`|Minimum percentage of rows containing texts that are valid names (not numeric)|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_name_pattern_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition text matching name pattern percent data quality check. - -??? example "Managing daily partition text matching name pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_text_matching_name_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_name_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_name_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_text_matching_name_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_name_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_text_matching_name_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_partition_text_matching_name_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_partition_text_matching_name_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_text_matching_name_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_partition_text_matching_name_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_partition_text_matching_name_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - -## monthly partition text matching name pattern percent - - -**Check description** - -Verifies that the percentage of texts matching the name regular expression does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_partition_text_matching_name_pattern_percent`|Minimum percentage of rows containing texts that are valid names (not numeric)|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*text_matching_name_pattern_percent*](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition text matching name pattern percent data quality check. - -??? example "Managing monthly partition text matching name pattern percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_text_matching_name_pattern_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_name_pattern_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_name_pattern_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_text_matching_name_pattern_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_name_pattern_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_text_matching_name_pattern_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_partition_text_matching_name_pattern_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_partition_text_matching_name_pattern_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_text_matching_name_pattern_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_partition_text_matching_name_pattern_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_partition_text_matching_name_pattern_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_text_matching_name_pattern_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [text_matching_name_pattern_percent](../../../reference/sensors/column/patterns-column-sensors.md#text-matching-name-pattern-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(CAST(analyzed_table.`target_column` AS STRING), r"^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches({{ lib.render_target_column('analyzed_table') }}, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN regexp_matches(analyzed_table."target_column", '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[[:alpha:] .''-]+$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP "^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([\\s-'.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([\\s-'.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$" - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '%[^ -~]%' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(CAST(analyzed_table."target_column" AS VARCHAR), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.])(\s?))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([''\s-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([''\s-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - - -## What's next -- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps -- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/column/patterns/text-not-matching-date-pattern-found.md b/docs/checks/column/patterns/text-not-matching-date-pattern-found.md index 00d6484709..3116e0b061 100644 --- a/docs/checks/column/patterns/text-not-matching-date-pattern-found.md +++ b/docs/checks/column/patterns/text-not-matching-date-pattern-found.md @@ -144,7 +144,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -165,7 +166,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -185,7 +187,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -206,7 +209,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -226,7 +230,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -247,7 +251,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -267,7 +271,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -288,7 +293,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -400,7 +406,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -428,7 +435,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -452,7 +460,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -473,7 +482,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -493,7 +503,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -514,7 +525,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -534,7 +546,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -555,7 +568,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -575,7 +589,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -596,7 +611,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -616,7 +632,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -644,7 +661,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -717,7 +735,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -737,7 +756,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -760,7 +780,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -780,7 +801,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -803,7 +825,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -823,7 +845,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -846,7 +868,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -866,7 +889,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -990,7 +1014,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1017,7 +1042,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1049,7 +1075,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1069,7 +1096,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -1092,7 +1120,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1112,7 +1141,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -1135,7 +1165,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1155,7 +1186,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -1178,7 +1210,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1198,7 +1231,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -1225,7 +1259,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1252,7 +1287,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1410,7 +1446,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -1431,7 +1468,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -1451,7 +1489,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1472,7 +1511,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -1492,7 +1532,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -1513,7 +1553,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -1533,7 +1573,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1554,7 +1595,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -1666,7 +1708,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1694,7 +1737,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1718,7 +1762,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1739,7 +1784,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -1759,7 +1805,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1780,7 +1827,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -1800,7 +1848,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1821,7 +1870,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -1841,7 +1891,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1862,7 +1913,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -1882,7 +1934,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1910,7 +1963,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1984,7 +2038,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2004,7 +2059,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -2027,7 +2083,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2047,7 +2104,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -2070,7 +2128,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -2090,7 +2148,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -2113,7 +2171,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2133,7 +2192,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -2257,7 +2317,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -2284,7 +2345,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -2316,7 +2378,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2336,7 +2399,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -2359,7 +2423,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2379,7 +2444,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -2402,7 +2468,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2422,7 +2489,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -2445,7 +2513,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2465,7 +2534,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -2492,7 +2562,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -2519,7 +2590,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -2677,7 +2749,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2698,7 +2771,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -2718,7 +2792,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2739,7 +2814,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -2759,7 +2835,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -2780,7 +2856,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -2800,7 +2876,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2821,7 +2898,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -2933,7 +3011,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -2961,7 +3040,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -2985,7 +3065,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3006,7 +3087,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -3026,7 +3108,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3047,7 +3130,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -3067,7 +3151,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3088,7 +3173,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -3108,7 +3194,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3129,7 +3216,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -3149,7 +3237,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3177,7 +3266,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -3251,7 +3341,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3271,7 +3362,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -3294,7 +3386,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3314,7 +3407,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -3337,7 +3431,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -3357,7 +3451,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -3380,7 +3474,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3400,7 +3495,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -3524,7 +3620,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3551,7 +3648,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -3583,7 +3681,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3603,7 +3702,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -3626,7 +3726,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3646,7 +3747,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -3669,7 +3771,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3689,7 +3792,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -3712,7 +3816,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3732,7 +3837,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -3759,7 +3865,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3786,7 +3893,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -3954,7 +4062,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3975,7 +4084,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -3999,7 +4109,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4020,7 +4131,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -4044,7 +4156,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -4065,7 +4177,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -4089,7 +4201,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4110,7 +4223,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -4236,7 +4350,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -4264,7 +4379,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -4294,7 +4410,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4315,7 +4432,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -4339,7 +4457,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4360,7 +4479,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -4384,7 +4504,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4405,7 +4526,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -4429,7 +4551,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4450,7 +4573,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -4476,7 +4600,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -4504,7 +4629,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -4594,7 +4720,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -4614,7 +4741,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -4639,7 +4767,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4659,7 +4788,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -4684,7 +4814,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -4704,7 +4834,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -4729,7 +4859,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4749,7 +4880,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -4881,7 +5013,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -4908,7 +5041,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -4944,7 +5078,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4964,7 +5099,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -4989,7 +5125,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5009,7 +5146,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -5034,7 +5172,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5054,7 +5193,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -5079,7 +5219,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5099,7 +5240,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -5126,7 +5268,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -5153,7 +5296,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -5325,7 +5469,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5346,7 +5491,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -5370,7 +5516,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5391,7 +5538,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -5415,7 +5563,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -5436,7 +5584,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -5460,7 +5608,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5481,7 +5630,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -5607,7 +5757,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -5635,7 +5786,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -5665,7 +5817,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5686,7 +5839,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -5710,7 +5864,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5731,7 +5886,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -5755,7 +5911,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5776,7 +5933,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -5800,7 +5958,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5821,7 +5980,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -5847,7 +6007,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -5875,7 +6036,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -5965,7 +6127,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5985,7 +6148,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -6010,7 +6174,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6030,7 +6195,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -6055,7 +6221,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -6075,7 +6241,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -6100,7 +6266,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6120,7 +6287,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -6252,7 +6420,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -6279,7 +6448,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -6315,7 +6485,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6335,7 +6506,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -6360,7 +6532,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6380,7 +6553,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE(analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -6405,7 +6579,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6425,7 +6600,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE(analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -6450,7 +6626,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6470,7 +6647,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, [target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, [target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -6497,7 +6675,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -6524,7 +6703,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END diff --git a/docs/checks/column/patterns/text-not-matching-date-pattern-percent.md b/docs/checks/column/patterns/text-not-matching-date-pattern-percent.md index a0a8d0653d..9118ec6a96 100644 --- a/docs/checks/column/patterns/text-not-matching-date-pattern-percent.md +++ b/docs/checks/column/patterns/text-not-matching-date-pattern-percent.md @@ -144,7 +144,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -165,7 +166,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -185,7 +187,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -206,7 +209,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -226,7 +230,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -247,7 +251,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -267,7 +271,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -288,7 +293,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -400,7 +406,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -428,7 +435,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -452,7 +460,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -473,7 +482,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -493,7 +503,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -514,7 +525,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -534,7 +546,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -555,7 +568,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -575,7 +589,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -596,7 +611,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -616,7 +632,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -644,7 +661,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -717,7 +735,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -737,7 +756,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -760,7 +780,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -780,7 +801,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -803,7 +825,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -823,7 +845,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -846,7 +868,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -866,7 +889,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -990,7 +1014,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1017,7 +1042,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1049,7 +1075,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1069,7 +1096,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -1092,7 +1120,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1112,7 +1141,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -1135,7 +1165,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1155,7 +1186,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -1178,7 +1210,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1198,7 +1231,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -1225,7 +1259,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1252,7 +1287,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1410,7 +1446,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -1431,7 +1468,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -1451,7 +1489,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1472,7 +1511,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -1492,7 +1532,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -1513,7 +1553,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -1533,7 +1573,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1554,7 +1595,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -1666,7 +1708,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1694,7 +1737,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1718,7 +1762,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1739,7 +1784,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -1759,7 +1805,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1780,7 +1827,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -1800,7 +1848,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1821,7 +1870,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -1841,7 +1891,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -1862,7 +1913,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -1882,7 +1934,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -1910,7 +1963,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -1984,7 +2038,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2004,7 +2059,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -2027,7 +2083,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2047,7 +2104,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -2070,7 +2128,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -2090,7 +2148,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -2113,7 +2171,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2133,7 +2192,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -2257,7 +2317,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -2284,7 +2345,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -2316,7 +2378,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2336,7 +2399,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -2359,7 +2423,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2379,7 +2444,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -2402,7 +2468,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2422,7 +2489,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -2445,7 +2513,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2465,7 +2534,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -2492,7 +2562,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -2519,7 +2590,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -2677,7 +2749,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2698,7 +2771,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -2718,7 +2792,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2739,7 +2814,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -2759,7 +2835,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -2780,7 +2856,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -2800,7 +2876,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -2821,7 +2898,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -2933,7 +3011,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -2961,7 +3040,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -2985,7 +3065,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3006,7 +3087,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -3026,7 +3108,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3047,7 +3130,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -3067,7 +3151,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3088,7 +3173,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -3108,7 +3194,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3129,7 +3216,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -3149,7 +3237,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3177,7 +3266,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -3251,7 +3341,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3271,7 +3362,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -3294,7 +3386,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3314,7 +3407,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -3337,7 +3431,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -3357,7 +3451,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -3380,7 +3474,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3400,7 +3495,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -3524,7 +3620,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3551,7 +3648,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -3583,7 +3681,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3603,7 +3702,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -3626,7 +3726,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3646,7 +3747,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -3669,7 +3771,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3689,7 +3792,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -3712,7 +3816,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3732,7 +3837,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -3759,7 +3865,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3786,7 +3893,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -3954,7 +4062,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3975,7 +4084,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -3999,7 +4109,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4020,7 +4131,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -4044,7 +4156,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -4065,7 +4177,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -4089,7 +4201,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4110,7 +4223,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -4236,7 +4350,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -4264,7 +4379,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -4294,7 +4410,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4315,7 +4432,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -4339,7 +4457,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4360,7 +4479,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -4384,7 +4504,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4405,7 +4526,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -4429,7 +4551,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4450,7 +4573,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -4476,7 +4600,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -4504,7 +4629,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -4594,7 +4720,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -4614,7 +4741,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -4639,7 +4767,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4659,7 +4788,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -4684,7 +4814,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -4704,7 +4834,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -4729,7 +4859,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4749,7 +4880,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -4881,7 +5013,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -4908,7 +5041,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -4944,7 +5078,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -4964,7 +5099,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -4989,7 +5125,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5009,7 +5146,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -5034,7 +5172,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5054,7 +5193,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -5079,7 +5219,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5099,7 +5240,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -5126,7 +5268,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -5153,7 +5296,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -5325,7 +5469,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5346,7 +5491,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -5370,7 +5516,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5391,7 +5538,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -5415,7 +5563,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -5436,7 +5584,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -5460,7 +5608,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5481,7 +5630,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -5607,7 +5757,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -5635,7 +5786,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -5665,7 +5817,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5686,7 +5839,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -5710,7 +5864,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5731,7 +5886,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -5755,7 +5911,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5776,7 +5933,8 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -5800,7 +5958,8 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -5821,7 +5980,8 @@ spec: WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -5847,7 +6007,8 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -5875,7 +6036,8 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -5965,7 +6127,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5985,7 +6148,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`target_column`) IS NULL THEN 1 ELSE 0 END @@ -6010,7 +6174,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6030,7 +6195,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TRY_TO_TIMESTAMP (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -6055,7 +6221,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -6075,7 +6241,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, 'yyyy-mm-dd') IS FALSE + WHEN REGEXP_MATCHES(analyzed_table."target_column"::VARCHAR, '^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])$') IS FALSE THEN 1 ELSE 0 END @@ -6100,7 +6266,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6120,7 +6287,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND STR_TO_DATE(analyzed_table.`target_column`, '%Y-%m-%d') IS NULL THEN 1 ELSE 0 END @@ -6252,7 +6420,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -6279,7 +6448,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END @@ -6315,7 +6485,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6335,7 +6506,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TO_DATE(analyzed_table."target_column"::VARCHAR, '%YYYY-%MM-%DD') IS NULL THEN 1 ELSE 0 END @@ -6360,7 +6532,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6380,7 +6553,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY_TO_DATE (analyzed_table."target_column", 'YYYY-MM-DD') IS NULL THEN 1 ELSE 0 END @@ -6405,7 +6579,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6425,7 +6600,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL + WHEN analyzed_table.`target_column` IS NOT NULL + AND TO_DATE (analyzed_table.`target_column`, 'yyyy-mm-dd') IS NULL THEN 1 ELSE 0 END @@ -6450,7 +6626,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -6470,7 +6647,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL + WHEN analyzed_table.[target_column] IS NOT NULL + AND TRY_CONVERT(DATETIME, analyzed_table.[target_column], 120) IS NULL THEN 1 ELSE 0 END @@ -6497,7 +6675,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -6524,7 +6703,8 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL + WHEN analyzed_table."target_column" IS NOT NULL + AND TRY(DATE_PARSE(analyzed_table."target_column", '%Y-%m-%d')) IS NULL THEN 1 ELSE 0 END diff --git a/docs/checks/column/patterns/text-not-matching-name-pattern-percent.md b/docs/checks/column/patterns/text-not-matching-name-pattern-percent.md index 8d00f864d7..adef1736de 100644 --- a/docs/checks/column/patterns/text-not-matching-name-pattern-percent.md +++ b/docs/checks/column/patterns/text-not-matching-name-pattern-percent.md @@ -261,7 +261,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -284,11 +284,11 @@ spec: CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -832,7 +832,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -854,11 +854,11 @@ Expand the *Configure with data grouping* section to see additional examples for CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -1511,7 +1511,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -1534,11 +1534,11 @@ spec: CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -2083,7 +2083,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -2105,11 +2105,11 @@ Expand the *Configure with data grouping* section to see additional examples for CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -2762,7 +2762,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -2785,11 +2785,11 @@ spec: CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -3334,7 +3334,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -3356,11 +3356,11 @@ Expand the *Configure with data grouping* section to see additional examples for CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -4035,7 +4035,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -4058,11 +4058,11 @@ spec: CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -4667,7 +4667,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -4689,11 +4689,11 @@ Expand the *Configure with data grouping* section to see additional examples for CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -5390,7 +5390,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -5413,11 +5413,11 @@ spec: CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END @@ -6022,7 +6022,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -6044,11 +6044,11 @@ Expand the *Configure with data grouping* section to see additional examples for CASE WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ + -])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ - ''\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ + -.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ - ''\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') + -.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') THEN 1 ELSE 0 END diff --git a/docs/checks/column/patterns/text-not-matching-regex-found.md b/docs/checks/column/patterns/text-not-matching-regex-found.md index a5c70fe128..9b8b68cfe2 100644 --- a/docs/checks/column/patterns/text-not-matching-regex-found.md +++ b/docs/checks/column/patterns/text-not-matching-regex-found.md @@ -144,9 +144,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -165,9 +165,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -185,9 +185,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -206,9 +206,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -226,9 +226,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -247,9 +247,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -267,9 +267,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -288,9 +288,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -308,9 +308,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -335,9 +335,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -359,9 +359,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -380,9 +380,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -400,9 +400,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -428,9 +428,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -452,9 +452,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -473,9 +473,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -493,9 +493,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -514,9 +514,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -534,9 +534,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -555,9 +555,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -575,9 +575,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -596,9 +596,9 @@ spec: WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -616,9 +616,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -644,9 +644,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -717,9 +717,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -737,9 +737,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -760,9 +760,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -780,9 +780,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -803,9 +803,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -823,9 +823,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -846,9 +846,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -866,9 +866,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -889,9 +889,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -915,9 +915,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -947,9 +947,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -967,9 +967,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -990,9 +990,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -1017,9 +1017,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value, @@ -1049,9 +1049,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1069,9 +1069,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -1092,9 +1092,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1112,9 +1112,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -1135,9 +1135,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1155,9 +1155,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -1178,9 +1178,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1198,9 +1198,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -1225,9 +1225,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1252,9 +1252,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -1410,9 +1410,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1431,9 +1431,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1451,9 +1451,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1472,9 +1472,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1492,9 +1492,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1513,9 +1513,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1533,9 +1533,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1554,9 +1554,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1574,9 +1574,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1601,9 +1601,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1625,9 +1625,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1646,9 +1646,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1666,9 +1666,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -1694,9 +1694,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -1718,9 +1718,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1739,9 +1739,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1759,9 +1759,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1780,9 +1780,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1800,9 +1800,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1821,9 +1821,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1841,9 +1841,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1862,9 +1862,9 @@ spec: WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1882,9 +1882,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1910,9 +1910,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -1984,9 +1984,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2004,9 +2004,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2027,9 +2027,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2047,9 +2047,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2070,9 +2070,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2090,9 +2090,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2113,9 +2113,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2133,9 +2133,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2156,9 +2156,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2182,9 +2182,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2214,9 +2214,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2234,9 +2234,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2257,9 +2257,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -2284,9 +2284,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value, @@ -2316,9 +2316,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2336,9 +2336,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2359,9 +2359,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2379,9 +2379,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2402,9 +2402,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2422,9 +2422,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2445,9 +2445,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2465,9 +2465,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2492,9 +2492,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2519,9 +2519,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -2677,9 +2677,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2698,9 +2698,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2718,9 +2718,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2739,9 +2739,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2759,9 +2759,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2780,9 +2780,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2800,9 +2800,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2821,9 +2821,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2841,9 +2841,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2868,9 +2868,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2892,9 +2892,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2913,9 +2913,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -2933,9 +2933,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -2961,9 +2961,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -2985,9 +2985,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3006,9 +3006,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3026,9 +3026,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3047,9 +3047,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3067,9 +3067,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3088,9 +3088,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3108,9 +3108,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3129,9 +3129,9 @@ spec: WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3149,9 +3149,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3177,9 +3177,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3251,9 +3251,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3271,9 +3271,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3294,9 +3294,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3314,9 +3314,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3337,9 +3337,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3357,9 +3357,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3380,9 +3380,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3400,9 +3400,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3423,9 +3423,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3449,9 +3449,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3481,9 +3481,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3501,9 +3501,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3524,9 +3524,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -3551,9 +3551,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value, @@ -3583,9 +3583,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3603,9 +3603,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3626,9 +3626,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3646,9 +3646,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3669,9 +3669,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3689,9 +3689,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3712,9 +3712,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3732,9 +3732,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3759,9 +3759,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3786,9 +3786,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3954,9 +3954,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -3975,9 +3975,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -3999,9 +3999,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4020,9 +4020,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4044,9 +4044,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4065,9 +4065,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4089,9 +4089,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4110,9 +4110,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4134,9 +4134,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4161,9 +4161,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4191,9 +4191,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4212,9 +4212,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4236,9 +4236,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -4264,9 +4264,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value, @@ -4294,9 +4294,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4315,9 +4315,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4339,9 +4339,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4360,9 +4360,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4384,9 +4384,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4405,9 +4405,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4429,9 +4429,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4450,9 +4450,9 @@ spec: WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4476,9 +4476,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4504,9 +4504,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4594,9 +4594,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4614,9 +4614,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4639,9 +4639,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4659,9 +4659,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4684,9 +4684,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4704,9 +4704,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4729,9 +4729,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4749,9 +4749,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4774,9 +4774,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4800,9 +4800,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4836,9 +4836,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4856,9 +4856,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4881,9 +4881,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -4908,9 +4908,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value, @@ -4944,9 +4944,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4964,9 +4964,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -4989,9 +4989,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5009,9 +5009,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5034,9 +5034,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5054,9 +5054,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5079,9 +5079,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5099,9 +5099,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5126,9 +5126,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5153,9 +5153,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5325,9 +5325,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5346,9 +5346,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5370,9 +5370,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5391,9 +5391,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5415,9 +5415,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5436,9 +5436,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5460,9 +5460,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5481,9 +5481,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5505,9 +5505,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5532,9 +5532,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5562,9 +5562,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5583,9 +5583,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5607,9 +5607,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -5635,9 +5635,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value, @@ -5665,9 +5665,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5686,9 +5686,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5710,9 +5710,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5731,9 +5731,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5755,9 +5755,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5776,9 +5776,9 @@ spec: WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5800,9 +5800,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5821,9 +5821,9 @@ spec: WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5847,9 +5847,9 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5875,9 +5875,9 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -5965,9 +5965,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -5985,9 +5985,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6010,9 +6010,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6030,9 +6030,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6055,9 +6055,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6075,9 +6075,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6100,9 +6100,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6120,9 +6120,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6145,9 +6145,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6171,9 +6171,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6207,9 +6207,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6227,9 +6227,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 0 - ELSE 1 + WHEN analyzed_table."target_column" !~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6252,9 +6252,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -6279,9 +6279,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value, @@ -6315,9 +6315,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6335,9 +6335,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6360,9 +6360,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6380,9 +6380,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6405,9 +6405,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6425,9 +6425,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.`target_column` REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6450,9 +6450,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6470,9 +6470,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table.[target_column]) = 0 THEN 0.0 ELSE SUM( CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 0 - ELSE 1 + WHEN NOT analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' + THEN 1 + ELSE 0 END ) END AS actual_value, @@ -6497,9 +6497,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -6524,9 +6524,9 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 1 + ELSE 0 END ) END AS actual_value, diff --git a/docs/checks/column/patterns/texts-matching-regex-percent.md b/docs/checks/column/patterns/texts-matching-regex-percent.md deleted file mode 100644 index 2c77a71409..0000000000 --- a/docs/checks/column/patterns/texts-matching-regex-percent.md +++ /dev/null @@ -1,6559 +0,0 @@ ---- -title: texts matching regex percent data quality checks ---- -# texts matching regex percent data quality checks - -This check validates text values using a pattern defined as a regular expression. - It measures the percentage of valid values and raises a data quality issue when the rate is below a threshold. - - -___ -The **texts matching regex percent** data quality check has the following variants for each -[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. - - -## profile texts matching regex percent - - -**Check description** - -Verifies that the percentage of strings matching the custom regular expression pattern does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`profile_texts_matching_regex_percent`|Minimum percent of rows containing texts values matching regex|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*texts_matching_regex_percent*](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile texts matching regex percent data quality check. - -??? example "Managing profile texts matching regex percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_texts_matching_regex_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_texts_matching_regex_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_texts_matching_regex_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_texts_matching_regex_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_texts_matching_regex_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_texts_matching_regex_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *profile_texts_matching_regex_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=profile_texts_matching_regex_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_texts_matching_regex_percent - ``` - - You can also run this check on all tables (and columns) on which the *profile_texts_matching_regex_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=profile_texts_matching_regex_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-17" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - profiling_checks: - patterns: - profile_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 29-34" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - profiling_checks: - patterns: - profile_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily texts matching regex percent - - -**Check description** - -Verifies that the percentage of strings matching the custom regular expression pattern does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_texts_matching_regex_percent`|Minimum percent of rows containing texts values matching regex|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*texts_matching_regex_percent*](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily texts matching regex percent data quality check. - -??? example "Managing daily texts matching regex percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_texts_matching_regex_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_texts_matching_regex_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_texts_matching_regex_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_texts_matching_regex_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_texts_matching_regex_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_texts_matching_regex_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_texts_matching_regex_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_texts_matching_regex_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_texts_matching_regex_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_texts_matching_regex_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_texts_matching_regex_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-18" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 30-35" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## monthly texts matching regex percent - - -**Check description** - -Verifies that the percentage of strings matching the custom regular expression pattern does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_texts_matching_regex_percent`|Minimum percent of rows containing texts values matching regex|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*texts_matching_regex_percent*](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly texts matching regex percent data quality check. - -??? example "Managing monthly texts matching regex percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_texts_matching_regex_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_texts_matching_regex_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_texts_matching_regex_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_texts_matching_regex_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_texts_matching_regex_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_texts_matching_regex_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_texts_matching_regex_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_texts_matching_regex_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_texts_matching_regex_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_texts_matching_regex_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_texts_matching_regex_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-18" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 30-35" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily partition texts matching regex percent - - -**Check description** - -Verifies that the percentage of strings matching the custom regular expression pattern does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_partition_texts_matching_regex_percent`|Minimum percent of rows containing texts values matching regex|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*texts_matching_regex_percent*](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition texts matching regex percent data quality check. - -??? example "Managing daily partition texts matching regex percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_texts_matching_regex_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_texts_matching_regex_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_texts_matching_regex_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_texts_matching_regex_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_texts_matching_regex_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_texts_matching_regex_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_partition_texts_matching_regex_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_partition_texts_matching_regex_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_texts_matching_regex_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_partition_texts_matching_regex_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_partition_texts_matching_regex_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-23" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 40-45" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - -## monthly partition texts matching regex percent - - -**Check description** - -Verifies that the percentage of strings matching the custom regular expression pattern does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_partition_texts_matching_regex_percent`|Minimum percent of rows containing texts values matching regex|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*texts_matching_regex_percent*](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition texts matching regex percent data quality check. - -??? example "Managing monthly partition texts matching regex percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_texts_matching_regex_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_texts_matching_regex_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_texts_matching_regex_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_texts_matching_regex_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_texts_matching_regex_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_texts_matching_regex_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_partition_texts_matching_regex_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_partition_texts_matching_regex_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_texts_matching_regex_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_partition_texts_matching_regex_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_partition_texts_matching_regex_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-23" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 40-45" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_texts_matching_regex_percent: - parameters: - regex: "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$" - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [texts_matching_regex_percent](../../../reference/sensors/column/patterns-column-sensors.md#texts-matching-regex-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(analyzed_table.`target_column`, r'^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex ) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` RLIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - - -## What's next -- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps -- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/column/patterns/texts-not-matching-regex-percent.md b/docs/checks/column/patterns/texts-not-matching-regex-percent.md index d1aa042393..e13cf1dc85 100644 --- a/docs/checks/column/patterns/texts-not-matching-regex-percent.md +++ b/docs/checks/column/patterns/texts-not-matching-regex-percent.md @@ -185,7 +185,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -226,7 +226,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -247,7 +247,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -308,7 +308,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -359,7 +359,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -380,7 +380,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -400,7 +400,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -452,7 +452,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -493,7 +493,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -534,7 +534,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -575,7 +575,7 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -616,7 +616,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -760,7 +760,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -803,7 +803,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -823,7 +823,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -889,7 +889,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -947,7 +947,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -967,7 +967,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -990,7 +990,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -1049,7 +1049,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1092,7 +1092,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1135,7 +1135,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1178,7 +1178,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1225,7 +1225,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -1451,7 +1451,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1492,7 +1492,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -1513,7 +1513,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -1574,7 +1574,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -1625,7 +1625,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -1646,7 +1646,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -1666,7 +1666,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -1718,7 +1718,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1759,7 +1759,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1800,7 +1800,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1841,7 +1841,7 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -1882,7 +1882,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -2027,7 +2027,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -2070,7 +2070,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -2090,7 +2090,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -2156,7 +2156,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -2214,7 +2214,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -2234,7 +2234,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -2257,7 +2257,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -2316,7 +2316,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -2359,7 +2359,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -2402,7 +2402,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -2445,7 +2445,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -2492,7 +2492,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -2718,7 +2718,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -2759,7 +2759,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -2780,7 +2780,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -2841,7 +2841,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -2892,7 +2892,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -2913,7 +2913,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -2933,7 +2933,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -2985,7 +2985,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3026,7 +3026,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3067,7 +3067,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3108,7 +3108,7 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3149,7 +3149,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -3294,7 +3294,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3337,7 +3337,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -3357,7 +3357,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -3423,7 +3423,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -3481,7 +3481,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -3501,7 +3501,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -3524,7 +3524,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -3583,7 +3583,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3626,7 +3626,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3669,7 +3669,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3712,7 +3712,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -3759,7 +3759,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -3999,7 +3999,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4044,7 +4044,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -4065,7 +4065,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -4134,7 +4134,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -4191,7 +4191,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -4212,7 +4212,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -4236,7 +4236,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -4294,7 +4294,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4339,7 +4339,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4384,7 +4384,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4429,7 +4429,7 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4476,7 +4476,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -4639,7 +4639,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4684,7 +4684,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -4704,7 +4704,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -4774,7 +4774,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -4836,7 +4836,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -4856,7 +4856,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -4881,7 +4881,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -4944,7 +4944,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4989,7 +4989,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5034,7 +5034,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5079,7 +5079,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5126,7 +5126,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -5370,7 +5370,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5415,7 +5415,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -5436,7 +5436,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -5505,7 +5505,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -5562,7 +5562,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -5583,7 +5583,7 @@ spec: WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -5607,7 +5607,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -5665,7 +5665,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5710,7 +5710,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5755,7 +5755,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5800,7 +5800,7 @@ spec: WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -5847,7 +5847,7 @@ spec: WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -6010,7 +6010,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -6055,7 +6055,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -6075,7 +6075,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS TRUE + WHEN REGEXP_MATCHES(analyzed_table."target_column", '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') IS FALSE THEN 1 ELSE 0 END @@ -6145,7 +6145,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -6207,7 +6207,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -6227,7 +6227,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS TRUE + WHEN analyzed_table."target_column" ~ '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$' IS FALSE THEN 1 ELSE 0 END @@ -6252,7 +6252,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -6315,7 +6315,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -6360,7 +6360,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -6405,7 +6405,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -6450,7 +6450,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -6497,7 +6497,7 @@ Expand the *Configure with data grouping* section to see additional examples for WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END diff --git a/docs/checks/column/patterns/valid-usa-phone-format-percent.md b/docs/checks/column/patterns/valid-usa-phone-format-percent.md deleted file mode 100644 index cb43e09338..0000000000 --- a/docs/checks/column/patterns/valid-usa-phone-format-percent.md +++ /dev/null @@ -1,6749 +0,0 @@ ---- -title: valid usa phone format percent data quality checks ---- -# valid usa phone format percent data quality checks - -This check validates the format of USA phone numbers inside text columns. - It measures the percentage of columns containing a phone number and raises a data quality issue when too many rows contain phone numbers. - - -___ -The **valid usa phone format percent** data quality check has the following variants for each -[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. - - -## profile valid usa phone format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`profile_valid_usa_phone_format_percent`|Minimum percentage of rows containing USA phone number values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_phone_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile valid usa phone format percent data quality check. - -??? example "Managing profile valid usa phone format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_valid_usa_phone_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_phone_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_phone_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_valid_usa_phone_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_phone_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_phone_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *profile_valid_usa_phone_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=profile_valid_usa_phone_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_valid_usa_phone_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *profile_valid_usa_phone_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=profile_valid_usa_phone_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-15" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - profiling_checks: - patterns: - profile_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 27-32" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - profiling_checks: - patterns: - profile_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily valid usa phone format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_valid_usa_phone_format_percent`|Minimum percentage of rows containing USA phone number values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_phone_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily valid usa phone format percent data quality check. - -??? example "Managing daily valid usa phone format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_valid_usa_phone_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_phone_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_phone_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_valid_usa_phone_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_phone_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_phone_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_valid_usa_phone_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_valid_usa_phone_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_valid_usa_phone_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_valid_usa_phone_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_valid_usa_phone_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## monthly valid usa phone format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_valid_usa_phone_format_percent`|Minimum percentage of rows containing USA phone number values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_phone_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly valid usa phone format percent data quality check. - -??? example "Managing monthly valid usa phone format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_valid_usa_phone_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_phone_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_phone_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_valid_usa_phone_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_phone_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_phone_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_valid_usa_phone_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_valid_usa_phone_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_valid_usa_phone_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_valid_usa_phone_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_valid_usa_phone_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily partition valid usa phone format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_partition_valid_usa_phone_format_percent`|Minimum percentage of rows containing USA phone number values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_phone_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition valid usa phone format percent data quality check. - -??? example "Managing daily partition valid usa phone format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_valid_usa_phone_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_phone_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_phone_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_valid_usa_phone_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_phone_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_phone_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_partition_valid_usa_phone_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_partition_valid_usa_phone_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_valid_usa_phone_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_partition_valid_usa_phone_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_partition_valid_usa_phone_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - -## monthly partition valid usa phone format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_partition_valid_usa_phone_format_percent`|Minimum percentage of rows containing USA phone number values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_phone_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)|:material-check-bold:| - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition valid usa phone format percent data quality check. - -??? example "Managing monthly partition valid usa phone format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_valid_usa_phone_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_phone_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_phone_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_valid_usa_phone_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_phone_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_phone_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_partition_valid_usa_phone_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_partition_valid_usa_phone_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_valid_usa_phone_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_partition_valid_usa_phone_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_partition_valid_usa_phone_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_valid_usa_phone_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_phone_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-phone-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^((((\\\\(\\\\+1\\\\)|(\\\\+1)|(\\\\([0][0][1]\\\\)|([0][0][1]))|\\\\(1\\\\)|(1))[\\\\s.-]?)?(\\\\(?[0-9]{3}\\\\)?[\\\\s.-]?)([0-9]{3}[\\\\s.-]?)([0-9]{4})))$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?[0-9]{3}\\)?[\\s.-]?)([0-9]{3}[\\s.-]?)([0-9]{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[[:space:].-]?)?(\(?\d{3}\)?[[:space:].-]?)(\d{3}[[:space:].-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace({{ lib.render_target_column('analyzed_table') }}, '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_SUBSTR(replace(replace(replace(analyzed_table."target_column", '(', ''), ')', ''), '-', ''), '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN ({{ lib.render_target_column('analyzed_table') }} REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE WHEN (analyzed_table."target_column" REGEXP '^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^((((\\(\\+1\\)|(\\+1)|(\\([0][0][1]\\)|([0][0][1]))|\\(1\\)|(1))[\\s.-]?)?(\\(?\\d{3}\\)?[\\s.-]?)(\\d{3}[\\s.-]?)(\\d{4})))$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][-.][0-9][0-9][0-9][-.][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '+1([0-9][0-9][0-9])[0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])[0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(+1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '(1)%[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' THEN 1 - WHEN analyzed_table.[target_column] LIKE '([0-9][0-9][0-9])-[0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^((((\(\+1\)|(\+1)|(\([0][0][1]\)|([0][0][1]))|\(1/)|(1))[\s.-]?)?(\(?\d{3}\)?[\s.-]?)(\d{3}[\s.-]?)(\d{4})))$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - - -## What's next -- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps -- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/column/patterns/valid-usa-zipcode-format-percent.md b/docs/checks/column/patterns/valid-usa-zipcode-format-percent.md deleted file mode 100644 index 54b89b0b3c..0000000000 --- a/docs/checks/column/patterns/valid-usa-zipcode-format-percent.md +++ /dev/null @@ -1,6749 +0,0 @@ ---- -title: valid usa zipcode format percent data quality checks ---- -# valid usa zipcode format percent data quality checks - -This check validates the format of a USA zip code inside text columns. - It measures the percentage of columns containing a valid zip code and raises a data quality issue when the rate is below a threshold. - - -___ -The **valid usa zipcode format percent** data quality check has the following variants for each -[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. - - -## profile valid usa zipcode format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`profile_valid_usa_zipcode_format_percent`|Minimum percentage of rows containing USA zip code values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_zipcode_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile valid usa zipcode format percent data quality check. - -??? example "Managing profile valid usa zipcode format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_valid_usa_zipcode_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_zipcode_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_zipcode_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_valid_usa_zipcode_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_zipcode_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_usa_zipcode_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *profile_valid_usa_zipcode_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=profile_valid_usa_zipcode_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_valid_usa_zipcode_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *profile_valid_usa_zipcode_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=profile_valid_usa_zipcode_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-15" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - profiling_checks: - patterns: - profile_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 27-32" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - profiling_checks: - patterns: - profile_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily valid usa zipcode format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_valid_usa_zipcode_format_percent`|Minimum percentage of rows containing USA zip code values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_zipcode_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily valid usa zipcode format percent data quality check. - -??? example "Managing daily valid usa zipcode format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_valid_usa_zipcode_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_zipcode_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_zipcode_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_valid_usa_zipcode_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_zipcode_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_usa_zipcode_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_valid_usa_zipcode_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_valid_usa_zipcode_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_valid_usa_zipcode_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_valid_usa_zipcode_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_valid_usa_zipcode_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## monthly valid usa zipcode format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_valid_usa_zipcode_format_percent`|Minimum percentage of rows containing USA zip code values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_zipcode_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly valid usa zipcode format percent data quality check. - -??? example "Managing monthly valid usa zipcode format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_valid_usa_zipcode_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_zipcode_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_zipcode_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_valid_usa_zipcode_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_zipcode_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_usa_zipcode_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_valid_usa_zipcode_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_valid_usa_zipcode_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_valid_usa_zipcode_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_valid_usa_zipcode_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_valid_usa_zipcode_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM ""."" original_table - ) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily partition valid usa zipcode format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_partition_valid_usa_zipcode_format_percent`|Minimum percentage of rows containing USA zip code values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_zipcode_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition valid usa zipcode format percent data quality check. - -??? example "Managing daily partition valid usa zipcode format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_valid_usa_zipcode_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_zipcode_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_zipcode_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_valid_usa_zipcode_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_zipcode_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_usa_zipcode_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_partition_valid_usa_zipcode_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_partition_valid_usa_zipcode_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_valid_usa_zipcode_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_partition_valid_usa_zipcode_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_partition_valid_usa_zipcode_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - -## monthly partition valid usa zipcode format percent - - -**Check description** - -Verifies that the percentage of valid USA phones number in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_partition_valid_usa_zipcode_format_percent`|Minimum percentage of rows containing USA zip code values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_usa_zipcode_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition valid usa zipcode format percent data quality check. - -??? example "Managing monthly partition valid usa zipcode format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_valid_usa_zipcode_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_zipcode_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_zipcode_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_valid_usa_zipcode_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_zipcode_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_usa_zipcode_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_partition_valid_usa_zipcode_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_partition_valid_usa_zipcode_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_valid_usa_zipcode_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_partition_valid_usa_zipcode_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_partition_valid_usa_zipcode_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_valid_usa_zipcode_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_usa_zipcode_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-usa-zipcode-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS( - CAST(analyzed_table.`target_column` AS STRING), - r"^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_MATCHES(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') IS TRUE - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), - '^[0-9]{5}(\-[0-9]{4})?$') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '^[0-9]{5}(\-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", - '^[0-9]{5}(?:-[0-9]{4})?$') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING({{ lib.render_target_column('analyzed_table') }} from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN SUBSTRING(analyzed_table."target_column" from - '^[0-9]{5}(?:-[0-9]{4})?$') IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9]{5}(/.D/:-[0-9]{4})?$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP( - CAST(analyzed_table.`target_column` AS STRING), - "^[0-9]{5}(?:-[0-9]{4})?$" - ) THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - {{ lib.render_target_column('analyzed_table') }} LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 0.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]' OR - analyzed_table.[target_column] LIKE '[0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]' - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END - AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 0.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE( - CAST(analyzed_table."target_column" AS VARCHAR), - '^[0-9]{5}(?:-[0-9]{4})?$' - ) THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END - AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - - -## What's next -- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps -- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/column/patterns/valid-uuid-format-percent.md b/docs/checks/column/patterns/valid-uuid-format-percent.md deleted file mode 100644 index b8e0d7c1ea..0000000000 --- a/docs/checks/column/patterns/valid-uuid-format-percent.md +++ /dev/null @@ -1,6479 +0,0 @@ ---- -title: valid uuid format percent data quality checks ---- -# valid uuid format percent data quality checks - -This check validates the format of UUID values in text columns. - It measures the percentage of valid UUIDs and raises a data quality issue when the rate is below a threshold. - - -___ -The **valid uuid format percent** data quality check has the following variants for each -[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. - - -## profile valid uuid format percent - - -**Check description** - -Verifies that the percentage of valid UUID in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`profile_valid_uuid_format_percent`|Minimum percentage of rows containing valid UUID values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_uuid_format_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile valid uuid format percent data quality check. - -??? example "Managing profile valid uuid format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_valid_uuid_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_uuid_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_uuid_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=profile_valid_uuid_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_uuid_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=profile_valid_uuid_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *profile_valid_uuid_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=profile_valid_uuid_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_valid_uuid_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *profile_valid_uuid_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=profile_valid_uuid_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-15" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - profiling_checks: - patterns: - profile_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM( - SELECT - original_table.* - FROM ""."" original_table) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 27-32" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - profiling_checks: - patterns: - profile_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily valid uuid format percent - - -**Check description** - -Verifies that the percentage of valid UUID in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_valid_uuid_format_percent`|Minimum percentage of rows containing valid UUID values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_uuid_format_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily valid uuid format percent data quality check. - -??? example "Managing daily valid uuid format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_valid_uuid_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_uuid_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_uuid_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_valid_uuid_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_uuid_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_valid_uuid_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_valid_uuid_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_valid_uuid_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_valid_uuid_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_valid_uuid_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_valid_uuid_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM( - SELECT - original_table.* - FROM ""."" original_table) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - daily: - patterns: - daily_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## monthly valid uuid format percent - - -**Check description** - -Verifies that the percentage of valid UUID in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_valid_uuid_format_percent`|Minimum percentage of rows containing valid UUID values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_uuid_format_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly valid uuid format percent data quality check. - -??? example "Managing monthly valid uuid format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_valid_uuid_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_uuid_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_uuid_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_valid_uuid_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_uuid_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_valid_uuid_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_valid_uuid_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_valid_uuid_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_valid_uuid_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_valid_uuid_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_valid_uuid_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="7-16" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `your-google-project-id`.``.`` AS analyzed_table - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM AS analyzed_table - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM `` AS analyzed_table - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM( - SELECT - original_table.* - FROM ""."" original_table) analyzed_table - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_postgresql_database".""."" AS analyzed_table - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_database".""."" original_table - ) analyzed_table - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_redshift_database".""."" AS analyzed_table - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM "your_snowflake_database".""."" AS analyzed_table - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value - FROM ``.`` AS analyzed_table - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value - FROM [your_sql_server_database].[].[] AS analyzed_table - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value - FROM ( - SELECT - original_table.* - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="5-13 28-33" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - monitoring_checks: - monthly: - patterns: - monthly_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM ""."" original_table) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2 - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2 - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2 - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state] - ORDER BY level_1, level_2 - , - - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2 - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2 - ORDER BY grouping_level_1, grouping_level_2 - ``` - -___ - - -## daily partition valid uuid format percent - - -**Check description** - -Verifies that the percentage of valid UUID in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`daily_partition_valid_uuid_format_percent`|Minimum percentage of rows containing valid UUID values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_uuid_format_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition valid uuid format percent data quality check. - -??? example "Managing daily partition valid uuid format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_valid_uuid_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_uuid_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_uuid_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=daily_partition_valid_uuid_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_uuid_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=daily_partition_valid_uuid_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *daily_partition_valid_uuid_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=daily_partition_valid_uuid_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_valid_uuid_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *daily_partition_valid_uuid_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=daily_partition_valid_uuid_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - daily: - patterns: - daily_partition_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE)) AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - CAST(analyzed_table."date_column" AS date) AS time_period, - TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - CAST(analyzed_table.`date_column` AS DATE) AS time_period, - TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - CAST(analyzed_table.[date_column] AS date) AS time_period, - CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) - ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - CAST(original_table."date_column" AS date) AS time_period, - CAST(CAST(original_table."date_column" AS date) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - -## monthly partition valid uuid format percent - - -**Check description** - -Verifies that the percentage of valid UUID in a text column does not fall below the minimum accepted percentage. - -|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| -|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| -|`monthly_partition_valid_uuid_format_percent`|Minimum percentage of rows containing valid UUID values|[patterns](../../../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Validity](../../../dqo-concepts/data-quality-dimensions.md#data-validity)|[*valid_uuid_format_percent*](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent)|[*min_percent*](../../../reference/rules/Comparison.md#min-percent)| | - -**Command-line examples** - -Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition valid uuid format percent data quality check. - -??? example "Managing monthly partition valid uuid format percent check from DQOps shell" - - === "Activate the check with a warning rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_valid_uuid_format_percent --enable-warning - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_uuid_format_percent --enable-warning - ``` - - Additional rule parameters are passed using the *-Wrule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_uuid_format_percent --enable-warning - -Wmin_percent=value - ``` - - - === "Activate the check with an error rule" - - Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, - providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. - - ``` - dqo> check activate -c=connection_name -t=schema_name.table_name -col=column_name -ch=monthly_partition_valid_uuid_format_percent --enable-error - ``` - - You can also use patterns to activate the check on all matching tables and columns. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_uuid_format_percent --enable-error - ``` - - Additional rule parameters are passed using the *-Erule_parameter_name=value*. - - ``` - dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -col=column_name -ch=monthly_partition_valid_uuid_format_percent --enable-error - -Emin_percent=value - ``` - - - === "Run all configured checks" - - Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. - The following example shows how to run the *monthly_partition_valid_uuid_format_percent* check on all tables and columns on a single data source. - - ``` - dqo> check run -c=data_source_name -ch=monthly_partition_valid_uuid_format_percent - ``` - - It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. - - ``` - dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_valid_uuid_format_percent - ``` - - You can also run this check on all tables (and columns) on which the *monthly_partition_valid_uuid_format_percent* check is enabled - using patterns to find tables. - - ``` - dqo> check run -c=connection_name -t=schema_prefix*.fact_* -col=column_name_* -ch=monthly_partition_valid_uuid_format_percent - ``` - - -**YAML configuration** - -The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. - - -```yaml hl_lines="12-21" -# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json -apiVersion: dqo/v1 -kind: table -spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - -``` - -??? info "Samples of generated SQL queries for each data source type" - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM( - SELECT - original_table.*, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc - ``` - - -Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). - -??? info "Configuration with data grouping" - - **Sample configuration with data grouping enabled (YAML)** - The sample below shows how to configure the data grouping and how it affects the generated SQL query. - - ```yaml hl_lines="10-4 38-43" - # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json - apiVersion: dqo/v1 - kind: table - spec: - timestamp_columns: - partition_by_column: date_column - incremental_time_window: - daily_partitioning_recent_days: 7 - monthly_partitioning_recent_months: 1 - default_grouping_name: group_by_country_and_state - groupings: - group_by_country_and_state: - level_1: - source: column_value - column: country - level_2: - source: column_value - column: state - columns: - target_column: - partitioned_checks: - monthly: - patterns: - monthly_partition_valid_uuid_format_percent: - warning: - min_percent: 100.0 - error: - min_percent: 99.0 - fatal: - min_percent: 95.0 - labels: - - This is the column that is analyzed for data quality issues - date_column: - labels: - - "date or datetime column used as a daily or monthly partitioning key, dates\ - \ (and times) are truncated to a day or a month by the sensor's query for\ - \ partitioned checks" - country: - labels: - - column used as the first grouping key - state: - labels: - - column used as the second grouping key - ``` - - Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the - [valid_uuid_format_percent](../../../reference/sensors/column/patterns-column-sensors.md#valid-uuid-format-percent) - [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). - - ??? example "BigQuery" - - === "Sensor template for BigQuery" - ```sql+jinja - {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for BigQuery" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`target_column` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, - TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc - FROM `your-google-project-id`.``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Databricks" - - === "Sensor template for Databricks" - ```sql+jinja - {% import '/dialects/databricks.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Databricks" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "DuckDB" - - === "Sensor template for DuckDB" - ```sql+jinja - {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for DuckDB" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "MySQL" - - === "Sensor template for MySQL" - ```sql+jinja - {% import '/dialects/mysql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), '[0-9a-fA-F]{8}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{4}[ \t\n\r\f\v-]?[0-9a-fA-F]{12}') }} - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for MySQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table.`target_column`, '[0-9a-fA-F]{8}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{4}[ - - -]?[0-9a-fA-F]{12}') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, - FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc - FROM `` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Oracle" - - === "Sensor template for Oracle" - ```sql+jinja - {% import '/dialects/oracle.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Oracle" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN REGEXP_LIKE(analyzed_table."target_column", '^[0-9A-F]{8}([ -]?)[0-9A-F]{4}([ -]?)[0-9A-F]{4}\1[0-9A-F]{4}\1[0-9A-F]{12}$','i') - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS time_period, - CAST(TRUNC(CAST(original_table."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM ""."" original_table) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "PostgreSQL" - - === "Sensor template for PostgreSQL" - ```sql+jinja - {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for PostgreSQL" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_postgresql_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Presto" - - === "Sensor template for Presto" - ```sql+jinja - {% import '/dialects/presto.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Presto" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_database".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Redshift" - - === "Sensor template for Redshift" - ```sql+jinja - {% import '/dialects/redshift.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Redshift" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" ~ '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc - FROM "your_redshift_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Snowflake" - - === "Sensor template for Snowflake" - ```sql+jinja - {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Snowflake" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table."target_column" REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table."target_column") - END AS actual_value, - analyzed_table."country" AS grouping_level_1, - analyzed_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, - TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc - FROM "your_snowflake_database".""."" AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "Spark" - - === "Sensor template for Spark" - ```sql+jinja - {% import '/dialects/spark.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Spark" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table.`target_column`) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN analyzed_table.`target_column` REGEXP '^[0-9a-fA-F]{8}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{4}[\\s-]?[0-9a-fA-F]{12}$' - THEN 1 - ELSE 0 - END - ) / COUNT(analyzed_table.`target_column`) - END AS actual_value, - analyzed_table.`country` AS grouping_level_1, - analyzed_table.`state` AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, - TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc - FROM ``.`` AS analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - ??? example "SQL Server" - - === "Sensor template for SQL Server" - ```sql+jinja - {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections('analyzed_table') }} - {{- lib.render_time_dimension_projection('analyzed_table') }} - FROM {{ lib.render_target_table() }} AS analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for SQL Server" - ```sql - SELECT - CASE - WHEN COUNT_BIG(analyzed_table.[target_column]) = 0 THEN 100.0 - ELSE 100.0 * SUM( - CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,analyzed_table.[target_column]) IS NOT NULL - THEN 1 - ELSE 0 - END - ) / COUNT_BIG(analyzed_table.[target_column]) - END AS actual_value, - analyzed_table.[country] AS grouping_level_1, - analyzed_table.[state] AS grouping_level_2, - DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, - CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc - FROM [your_sql_server_database].[].[] AS analyzed_table - GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) - ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) - - - ``` - ??? example "Trino" - - === "Sensor template for Trino" - ```sql+jinja - {% import '/dialects/trino.sql.jinja2' as lib with context -%} - SELECT - CASE - WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) - END AS actual_value - {{- lib.render_data_grouping_projections_reference('analyzed_table') }} - {{- lib.render_time_dimension_projection_reference('analyzed_table') }} - FROM ( - SELECT - original_table.* - {{- lib.render_data_grouping_projections('original_table') }} - {{- lib.render_time_dimension_projection('original_table') }} - FROM {{ lib.render_target_table() }} original_table - {{- lib.render_where_clause(table_alias_prefix='original_table') }} - ) analyzed_table - {{- lib.render_where_clause() -}} - {{- lib.render_group_by() -}} - {{- lib.render_order_by() -}} - ``` - === "Rendered SQL for Trino" - ```sql - SELECT - CASE - WHEN COUNT(analyzed_table."target_column") = 0 THEN 100.0 - ELSE CAST(100.0 * SUM( - CASE - WHEN REGEXP_LIKE(TRY_CAST(analyzed_table."target_column" AS VARCHAR), '^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$') - THEN 1 - ELSE 0 - END - ) AS DOUBLE) / COUNT(analyzed_table."target_column") - END AS actual_value, - - analyzed_table.grouping_level_1, - - analyzed_table.grouping_level_2 - , - time_period, - time_period_utc - FROM ( - SELECT - original_table.*, - original_table."country" AS grouping_level_1, - original_table."state" AS grouping_level_2, - DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS time_period, - CAST(DATE_TRUNC('MONTH', CAST(original_table."date_column" AS date)) AS TIMESTAMP) AS time_period_utc - FROM "your_trino_catalog".""."" original_table - ) analyzed_table - GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc - ``` - -___ - - - -## What's next -- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps -- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/column/whitespace/whitespace-text-found.md b/docs/checks/column/whitespace/whitespace-text-found.md index a40b2ea88e..1b6a94dd7a 100644 --- a/docs/checks/column/whitespace/whitespace-text-found.md +++ b/docs/checks/column/whitespace/whitespace-text-found.md @@ -295,8 +295,7 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -320,8 +319,7 @@ spec: SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -543,8 +541,8 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -563,8 +561,8 @@ spec: SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -841,8 +839,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -865,8 +862,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -1111,8 +1107,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -1130,8 +1126,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -1491,8 +1487,7 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -1516,8 +1511,7 @@ spec: SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -1739,8 +1733,8 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -1759,8 +1753,8 @@ spec: SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -2038,8 +2032,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2062,8 +2055,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -2308,8 +2300,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -2327,8 +2319,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -2688,8 +2680,7 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2713,8 +2704,7 @@ spec: SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -2936,8 +2926,8 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -2956,8 +2946,8 @@ spec: SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -3235,8 +3225,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3259,8 +3248,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -3505,8 +3493,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -3524,8 +3512,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -3911,8 +3899,7 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3936,8 +3923,7 @@ spec: SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -4187,8 +4173,8 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -4207,8 +4193,8 @@ spec: SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -4516,8 +4502,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -4540,8 +4525,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -4802,8 +4786,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -4821,8 +4805,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -5212,8 +5196,7 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5237,8 +5220,7 @@ spec: SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -5488,8 +5470,8 @@ spec: SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -5508,8 +5490,8 @@ spec: SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END @@ -5817,8 +5799,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5841,8 +5822,7 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -6103,8 +6083,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -6122,8 +6102,8 @@ Expand the *Configure with data grouping* section to see additional examples for SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 - AND TRIM(analyzed_table.[target_column]) = '' + AND DATALENGTH(analyzed_table.[target_column]) <> 0 + AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 END diff --git a/docs/checks/column/whitespace/whitespace-text-percent.md b/docs/checks/column/whitespace/whitespace-text-percent.md index 3d98ff54f4..898d09bd92 100644 --- a/docs/checks/column/whitespace/whitespace-text-percent.md +++ b/docs/checks/column/whitespace/whitespace-text-percent.md @@ -321,8 +321,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -349,8 +348,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -606,7 +604,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -629,7 +627,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -940,8 +938,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -967,8 +964,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -1247,7 +1243,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -1269,7 +1265,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -1663,8 +1659,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -1691,8 +1686,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -1948,7 +1942,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -1971,7 +1965,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -2283,8 +2277,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2310,8 +2303,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -2590,7 +2582,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -2612,7 +2604,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -3006,8 +2998,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3034,8 +3025,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -3291,7 +3281,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -3314,7 +3304,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -3626,8 +3616,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3653,8 +3642,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -3933,7 +3921,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -3955,7 +3943,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -4375,8 +4363,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -4403,8 +4390,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -4688,7 +4674,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -4711,7 +4697,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -5053,8 +5039,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5080,8 +5065,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -5376,7 +5360,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -5398,7 +5382,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -5822,8 +5806,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -5850,8 +5833,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -6135,7 +6117,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -6158,7 +6140,7 @@ spec: ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 @@ -6500,8 +6482,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -6527,8 +6508,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table."target_column" IS NOT NULL - AND analyzed_table."target_column" <> '' - AND TRIM(analyzed_table."target_column") = '' + AND TRIM(analyzed_table."target_column") IS NULL THEN 1 ELSE 0 END @@ -6823,7 +6803,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 @@ -6845,7 +6825,7 @@ Expand the *Configure with data grouping* section to see additional examples for ELSE 100.0 * SUM( CASE WHEN analyzed_table.[target_column] IS NOT NULL - AND LEN(analyzed_table.[target_column]) <> 0 + AND DATALENGTH(analyzed_table.[target_column]) <> 0 AND TRIM(analyzed_table.[target_column]) = '' THEN 1 ELSE 0 diff --git a/docs/checks/index.md b/docs/checks/index.md index c6a30eecb5..33c28a75ac 100644 --- a/docs/checks/index.md +++ b/docs/checks/index.md @@ -169,6 +169,24 @@ A table-level check that calculates the maximum difference in days between inges +## table-level uniqueness checks + + +### [duplicate record count](./table/uniqueness/duplicate-record-count.md) +This check counts duplicate records values. It raises a data quality issue when the number of duplicates is above a minimum accepted value. + The default configuration detects duplicate rows by enforcing that the *min_count* of duplicates is zero. + + + +### [duplicate record percent](./table/uniqueness/duplicate-record-percent.md) +This check measures the percentage of duplicate records values. It raises a data quality issue when the percentage of duplicates is above a minimum accepted value. + The default threshold is 0% duplicate values. + + + + + + ## table-level volume checks Evaluates the overall quality of the table by verifying the number of rows. @@ -290,6 +308,9 @@ This check compares the current table volume (the row count) to the row count 30 + + + diff --git a/docs/checks/table/index.md b/docs/checks/table/index.md index 82cc9e48ef..715b637053 100644 --- a/docs/checks/table/index.md +++ b/docs/checks/table/index.md @@ -169,6 +169,24 @@ A table-level check that calculates the maximum difference in days between inges +## table-level uniqueness checks + + +### [duplicate record count](./uniqueness/duplicate-record-count.md) +This check counts duplicate records values. It raises a data quality issue when the number of duplicates is above a minimum accepted value. + The default configuration detects duplicate rows by enforcing that the *min_count* of duplicates is zero. + + + +### [duplicate record percent](./uniqueness/duplicate-record-percent.md) +This check measures the percentage of duplicate records values. It raises a data quality issue when the percentage of duplicates is above a minimum accepted value. + The default threshold is 0% duplicate values. + + + + + + ## table-level volume checks Evaluates the overall quality of the table by verifying the number of rows. diff --git a/docs/checks/table/uniqueness/duplicate-record-count.md b/docs/checks/table/uniqueness/duplicate-record-count.md new file mode 100644 index 0000000000..7af13c4b31 --- /dev/null +++ b/docs/checks/table/uniqueness/duplicate-record-count.md @@ -0,0 +1,7780 @@ +--- +title: duplicate record count data quality checks +--- +# duplicate record count data quality checks + +This check counts duplicate records values. It raises a data quality issue when the number of duplicates is above a minimum accepted value. + The default configuration detects duplicate rows by enforcing that the *min_count* of duplicates is zero. + + +___ +The **duplicate record count** data quality check has the following variants for each +[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. + + +## profile duplicate record count + + +**Check description** + +Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`profile_duplicate_record_count`|Maximum count of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_count*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count)|[*max_count*](../../../reference/rules/Comparison.md#max-count)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile duplicate record count data quality check. + +??? example "Managing profile duplicate record count check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=profile_duplicate_record_count --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_count --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_count --enable-warning + -Wmax_count=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=profile_duplicate_record_count --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_count --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_count --enable-error + -Emax_count=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *profile_duplicate_record_count* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=profile_duplicate_record_count + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_duplicate_record_count + ``` + + You can also run this check on all tables on which the *profile_duplicate_record_count* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_count + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="5-8" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + profiling_checks: + uniqueness: + profile_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: {} + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at] + ) grouping_table + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="5-13 28-33" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + profiling_checks: + uniqueness: + profile_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2 + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state] + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state] + ORDER BY level_1, level_2 + , + + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + +___ + + +## daily duplicate record count + + +**Check description** + +Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`daily_duplicate_record_count`|Maximum count of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_count*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count)|[*max_count*](../../../reference/rules/Comparison.md#max-count)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily duplicate record count data quality check. + +??? example "Managing daily duplicate record count check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_duplicate_record_count --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_count --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_count --enable-warning + -Wmax_count=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_duplicate_record_count --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_count --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_count --enable-error + -Emax_count=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *daily_duplicate_record_count* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=daily_duplicate_record_count + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_duplicate_record_count + ``` + + You can also run this check on all tables on which the *daily_duplicate_record_count* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_count + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="5-9" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + monitoring_checks: + daily: + uniqueness: + daily_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: {} + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at] + ) grouping_table + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="5-13 29-34" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + monitoring_checks: + daily: + uniqueness: + daily_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2 + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state] + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state] + ORDER BY level_1, level_2 + , + + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + +___ + + +## monthly duplicate record count + + +**Check description** + +Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`monthly_duplicate_record_count`|Maximum count of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_count*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count)|[*max_count*](../../../reference/rules/Comparison.md#max-count)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly duplicate record count data quality check. + +??? example "Managing monthly duplicate record count check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_duplicate_record_count --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_count --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_count --enable-warning + -Wmax_count=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_duplicate_record_count --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_count --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_count --enable-error + -Emax_count=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *monthly_duplicate_record_count* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=monthly_duplicate_record_count + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_duplicate_record_count + ``` + + You can also run this check on all tables on which the *monthly_duplicate_record_count* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_count + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="5-9" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + monitoring_checks: + monthly: + uniqueness: + monthly_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: {} + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at] + ) grouping_table + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + FROM ( + SELECT COUNT(*) AS duplicated_count + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="5-13 29-34" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + monitoring_checks: + monthly: + uniqueness: + monthly_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2 + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state] + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state] + ORDER BY level_1, level_2 + , + + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + +___ + + +## daily partition duplicate record count + + +**Check description** + +Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`daily_partition_duplicate_record_count`|Maximum count of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_count*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count)|[*max_count*](../../../reference/rules/Comparison.md#max-count)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition duplicate record count data quality check. + +??? example "Managing daily partition duplicate record count check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_partition_duplicate_record_count --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_count --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_count --enable-warning + -Wmax_count=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_partition_duplicate_record_count --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_count --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_count --enable-error + -Emax_count=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *daily_partition_duplicate_record_count* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=daily_partition_duplicate_record_count + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_duplicate_record_count + ``` + + You can also run this check on all tables on which the *daily_partition_duplicate_record_count* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_count + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="10-14" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + partitioned_checks: + daily: + uniqueness: + daily_partition_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table."date_column" AS date) AS time_period, + TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + CAST(analyzed_table.[date_column] AS date) AS time_period, + CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ) grouping_table + GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ORDER BY CAST(analyzed_table.[date_column] AS date) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="10-4 39-44" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + partitioned_checks: + daily: + uniqueness: + daily_partition_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2, + CAST(analyzed_table.[date_column] AS date) AS time_period, + CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + +___ + + +## monthly partition duplicate record count + + +**Check description** + +Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`monthly_partition_duplicate_record_count`|Maximum count of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_count*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count)|[*max_count*](../../../reference/rules/Comparison.md#max-count)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition duplicate record count data quality check. + +??? example "Managing monthly partition duplicate record count check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_partition_duplicate_record_count --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_count --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_count --enable-warning + -Wmax_count=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_partition_duplicate_record_count --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_count --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_count --enable-error + -Emax_count=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *monthly_partition_duplicate_record_count* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=monthly_partition_duplicate_record_count + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_duplicate_record_count + ``` + + You can also run this check on all tables on which the *monthly_partition_duplicate_record_count* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_count + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="10-14" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + partitioned_checks: + monthly: + uniqueness: + monthly_partition_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, + TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, + CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ) grouping_table + GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="10-4 39-44" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + partitioned_checks: + monthly: + uniqueness: + monthly_partition_duplicate_record_count: + parameters: + columns: + - id + - created_at + warning: + max_count: 0 + error: + max_count: 10 + fatal: + max_count: 100 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_count](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-count) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, + TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2, + DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, + CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS duplicated_count, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + +___ + + + +## What's next +- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps +- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/table/uniqueness/duplicate-record-percent.md b/docs/checks/table/uniqueness/duplicate-record-percent.md new file mode 100644 index 0000000000..9105847284 --- /dev/null +++ b/docs/checks/table/uniqueness/duplicate-record-percent.md @@ -0,0 +1,7740 @@ +--- +title: duplicate record percent data quality checks +--- +# duplicate record percent data quality checks + +This check measures the percentage of duplicate records values. It raises a data quality issue when the percentage of duplicates is above a minimum accepted value. + The default threshold is 0% duplicate values. + + +___ +The **duplicate record percent** data quality check has the following variants for each +[type of data quality](../../../dqo-concepts/definition-of-data-quality-checks/index.md#types-of-checks) checks supported by DQOps. + + +## profile duplicate record percent + + +**Check description** + +Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`profile_duplicate_record_percent`|Maximum percentage of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)| |[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_percent*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent)|[*max_percent*](../../../reference/rules/Comparison.md#max-percent)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the profile duplicate record percent data quality check. + +??? example "Managing profile duplicate record percent check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=profile_duplicate_record_percent --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_percent --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_percent --enable-warning + -Wmax_percent=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=profile_duplicate_record_percent --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_percent --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_percent --enable-error + -Emax_percent=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *profile_duplicate_record_percent* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=profile_duplicate_record_percent + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=profile_duplicate_record_percent + ``` + + You can also run this check on all tables on which the *profile_duplicate_record_percent* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=profile_duplicate_record_percent + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="5-8" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + profiling_checks: + uniqueness: + profile_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: {} + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at] + ) grouping_table + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="5-13 28-33" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + profiling_checks: + uniqueness: + profile_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2 + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state] + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state] + ORDER BY level_1, level_2 + , + + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + +___ + + +## daily duplicate record percent + + +**Check description** + +Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`daily_duplicate_record_percent`|Maximum percentage of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|daily|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_percent*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent)|[*max_percent*](../../../reference/rules/Comparison.md#max-percent)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily duplicate record percent data quality check. + +??? example "Managing daily duplicate record percent check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_duplicate_record_percent --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_percent --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_percent --enable-warning + -Wmax_percent=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_duplicate_record_percent --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_percent --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_percent --enable-error + -Emax_percent=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *daily_duplicate_record_percent* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=daily_duplicate_record_percent + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_duplicate_record_percent + ``` + + You can also run this check on all tables on which the *daily_duplicate_record_percent* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=daily_duplicate_record_percent + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="5-9" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + monitoring_checks: + daily: + uniqueness: + daily_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: {} + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at] + ) grouping_table + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="5-13 29-34" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + monitoring_checks: + daily: + uniqueness: + daily_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2 + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state] + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state] + ORDER BY level_1, level_2 + , + + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + +___ + + +## monthly duplicate record percent + + +**Check description** + +Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`monthly_duplicate_record_percent`|Maximum percentage of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|monthly|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_percent*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent)|[*max_percent*](../../../reference/rules/Comparison.md#max-percent)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly duplicate record percent data quality check. + +??? example "Managing monthly duplicate record percent check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_duplicate_record_percent --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_percent --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_percent --enable-warning + -Wmax_percent=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_duplicate_record_percent --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_percent --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_percent --enable-error + -Emax_percent=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *monthly_duplicate_record_percent* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=monthly_duplicate_record_percent + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_duplicate_record_percent + ``` + + You can also run this check on all tables on which the *monthly_duplicate_record_percent* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_duplicate_record_percent + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="5-9" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + monitoring_checks: + monthly: + uniqueness: + monthly_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: {} + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at" + ) grouping_table + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at` + ) grouping_table + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at] + ) grouping_table + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records + FROM ( + SELECT + "id", "created_at" + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at" + ) grouping_table + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="5-13 29-34" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + monitoring_checks: + monthly: + uniqueness: + monthly_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2 + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2 + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2 + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state] + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state] + ORDER BY level_1, level_2 + , + + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2 + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2 + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2 + ORDER BY grouping_level_1, grouping_level_2 + ``` + +___ + + +## daily partition duplicate record percent + + +**Check description** + +Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`daily_partition_duplicate_record_percent`|Maximum percentage of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|daily|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_percent*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent)|[*max_percent*](../../../reference/rules/Comparison.md#max-percent)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the daily partition duplicate record percent data quality check. + +??? example "Managing daily partition duplicate record percent check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_partition_duplicate_record_percent --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_percent --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_percent --enable-warning + -Wmax_percent=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=daily_partition_duplicate_record_percent --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_percent --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_percent --enable-error + -Emax_percent=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *daily_partition_duplicate_record_percent* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=daily_partition_duplicate_record_percent + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=daily_partition_duplicate_record_percent + ``` + + You can also run this check on all tables on which the *daily_partition_duplicate_record_percent* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=daily_partition_duplicate_record_percent + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="10-14" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + partitioned_checks: + daily: + uniqueness: + daily_partition_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + CAST(analyzed_table."date_column" AS date) AS time_period, + TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records, + CAST(analyzed_table.[date_column] AS date) AS time_period, + CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ) grouping_table + GROUP BY CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ORDER BY CAST(analyzed_table.[date_column] AS date) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="10-4 39-44" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + partitioned_checks: + daily: + uniqueness: + daily_partition_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-%d 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + CAST((CAST(analyzed_table."date_column" AS date)) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + CAST(analyzed_table."date_column" AS date) AS time_period, + TO_TIMESTAMP(CAST(analyzed_table."date_column" AS date)) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + CAST(analyzed_table.`date_column` AS DATE) AS time_period, + TIMESTAMP(CAST(analyzed_table.`date_column` AS DATE)) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2, + CAST(analyzed_table.[date_column] AS date) AS time_period, + CAST((CAST(analyzed_table.[date_column] AS date)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state], CAST(analyzed_table.[date_column] AS date), CAST(analyzed_table.[date_column] AS date) + ORDER BY level_1, level_2CAST(analyzed_table.[date_column] AS date) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + CAST(analyzed_table_nested."date_column" AS date) AS time_period, + CAST(CAST(analyzed_table_nested."date_column" AS date) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + +___ + + +## monthly partition duplicate record percent + + +**Check description** + +Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage. + +|Data quality check name|Friendly name|Category|Check type|Time scale|Quality dimension|Sensor definition|Quality rule|Standard| +|-----------------------|-------------|--------|----------|----------|-----------------|-----------------|------------|--------| +|`monthly_partition_duplicate_record_percent`|Maximum percentage of duplicate records|[uniqueness](../../../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|monthly|[Uniqueness](../../../dqo-concepts/data-quality-dimensions.md#data-uniqueness)|[*duplicate_record_percent*](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent)|[*max_percent*](../../../reference/rules/Comparison.md#max-percent)|:material-check-bold:| + +**Command-line examples** + +Please expand the section below to see the [DQOps command-line](../../../dqo-concepts/command-line-interface.md) examples to run or activate the monthly partition duplicate record percent data quality check. + +??? example "Managing monthly partition duplicate record percent check from DQOps shell" + + === "Activate the check with a warning rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the warning rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_partition_duplicate_record_percent --enable-warning + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_percent --enable-warning + ``` + + Additional rule parameters are passed using the *-Wrule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_percent --enable-warning + -Wmax_percent=value + ``` + + + === "Activate the check with an error rule" + + Activate this data quality using the [check activate](../../../command-line-interface/check.md#dqo-check-activate) CLI command, + providing the connection name, table name, check name, and all other filters. Activates the error rule with the default parameters. + + ``` + dqo> check activate -c=connection_name -t=schema_name.table_name -ch=monthly_partition_duplicate_record_percent --enable-error + ``` + + You can also use patterns to activate the check on all matching tables and columns. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_percent --enable-error + ``` + + Additional rule parameters are passed using the *-Erule_parameter_name=value*. + + ``` + dqo> check activate -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_percent --enable-error + -Emax_percent=value + ``` + + + === "Run all configured checks" + + Run this data quality check using the [check run](../../../command-line-interface/check.md#dqo-check-run) CLI command by providing the check name and all other targeting filters. + The following example shows how to run the *monthly_partition_duplicate_record_percent* check on all tables on a single data source. + + ``` + dqo> check run -c=data_source_name -ch=monthly_partition_duplicate_record_percent + ``` + + It is also possible to run this check on a specific connection and table. In order to do this, use the connection name and the full table name parameters. + + ``` + dqo> check run -c=connection_name -t=schema_name.table_name -ch=monthly_partition_duplicate_record_percent + ``` + + You can also run this check on all tables on which the *monthly_partition_duplicate_record_percent* check is enabled + using patterns to find tables. + + ``` + dqo> check run -c=connection_name -t=schema_prefix*.fact_* -ch=monthly_partition_duplicate_record_percent + ``` + + +**YAML configuration** + +The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. + + +```yaml hl_lines="10-14" +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + partitioned_checks: + monthly: + uniqueness: + monthly_partition_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + +``` + +??? info "Samples of generated SQL queries for each data source type" + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [data quality sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, + TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records, + DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, + CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ) grouping_table + GROUP BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ORDER BY DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", time_period, time_period_utc + ) grouping_table + GROUP BY time_period, time_period_utc + ORDER BY time_period, time_period_utc + ``` + + +Expand the *Configure with data grouping* section to see additional examples for configuring this data quality checks to use data grouping (GROUP BY). + +??? info "Configuration with data grouping" + + **Sample configuration with data grouping enabled (YAML)** + The sample below shows how to configure the data grouping and how it affects the generated SQL query. + + ```yaml hl_lines="10-4 39-44" + # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json + apiVersion: dqo/v1 + kind: table + spec: + timestamp_columns: + partition_by_column: date_column + incremental_time_window: + daily_partitioning_recent_days: 7 + monthly_partitioning_recent_months: 1 + default_grouping_name: group_by_country_and_state + groupings: + group_by_country_and_state: + level_1: + source: column_value + column: country + level_2: + source: column_value + column: state + partitioned_checks: + monthly: + uniqueness: + monthly_partition_duplicate_record_percent: + parameters: + columns: + - id + - created_at + warning: + max_percent: 0.0 + error: + max_percent: 1.0 + fatal: + max_percent: 5.0 + columns: + date_column: + labels: + - "date or datetime column used as a daily or monthly partitioning key, dates\ + \ (and times) are truncated to a day or a month by the sensor's query for\ + \ partitioned checks" + country: + labels: + - column used as the first grouping key + state: + labels: + - column used as the second grouping key + ``` + + Please expand the database engine name section to see the SQL query rendered by a Jinja2 template for the + [duplicate_record_percent](../../../reference/sensors/table/uniqueness-table-sensors.md#duplicate-record-percent) + [sensor](../../../dqo-concepts/definition-of-data-quality-sensors.md). + + ??? example "BigQuery" + + === "Sensor template for BigQuery" + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for BigQuery" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH) AS time_period, + TIMESTAMP(DATE_TRUNC(CAST(analyzed_table.`date_column` AS DATE), MONTH)) AS time_period_utc + FROM `your-google-project-id`.``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Databricks" + + === "Sensor template for Databricks" + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Databricks" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + grouping_table.grouping_level_1, + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "DuckDB" + + === "Sensor template for DuckDB" + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for DuckDB" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM AS analyzed_table + WHERE (COALESCE(CAST( "id" AS VARCHAR), CAST( "created_at" AS VARCHAR)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "MySQL" + + === "Sensor template for MySQL" + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for MySQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00') AS time_period, + FROM_UNIXTIME(UNIX_TIMESTAMP(DATE_FORMAT(analyzed_table.`date_column`, '%Y-%m-01 00:00:00'))) AS time_period_utc + FROM `` AS analyzed_table + WHERE (COALESCE(`id`, `created_at`) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Oracle" + + === "Sensor template for Oracle" + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Oracle" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS time_period, + CAST(TRUNC(CAST(analyzed_table_nested."date_column" AS DATE), 'MONTH') AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM ""."" analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR(4000)), CAST("created_at" AS VARCHAR(4000))) IS NOT NULL) + ) analyzed_table + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "PostgreSQL" + + === "Sensor template for PostgreSQL" + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for PostgreSQL" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_postgresql_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Presto" + + === "Sensor template for Presto" + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Presto" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_database".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Redshift" + + === "Sensor template for Redshift" + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Redshift" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + CAST((DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS TIMESTAMP WITH TIME ZONE) AS time_period_utc + FROM "your_redshift_database".""."" AS analyzed_table + WHERE (COALESCE("id"::VARCHAR, "created_at"::VARCHAR) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Snowflake" + + === "Sensor template for Snowflake" + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Snowflake" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + analyzed_table."country" AS grouping_level_1, + analyzed_table."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date)) AS time_period, + TO_TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table."date_column" AS date))) AS time_period_utc + FROM "your_snowflake_database".""."" AS analyzed_table + WHERE (COALESCE(CAST("id" AS STRING), CAST("created_at" AS STRING)) IS NOT NULL) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "Spark" + + === "Sensor template for Spark" + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Spark" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY `id`, `created_at`) AS distinct_records, + analyzed_table.`country` AS grouping_level_1, + analyzed_table.`state` AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE)) AS time_period, + TIMESTAMP(DATE_TRUNC('MONTH', CAST(analyzed_table.`date_column` AS DATE))) AS time_period_utc + FROM ``.`` AS analyzed_table + WHERE (COALESCE(CAST(`id` AS STRING), CAST(`created_at` AS STRING)) IS NOT NULL) + GROUP BY `id`, `created_at`, grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + ??? example "SQL Server" + + === "Sensor template for SQL Server" + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for SQL Server" + ```sql + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2, + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY [id], [created_at]) AS distinct_records, + analyzed_table.[country] AS grouping_level_1, + analyzed_table.[state] AS grouping_level_2, + DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) AS time_period, + CAST((DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1)) AS DATETIME) AS time_period_utc + FROM [your_sql_server_database].[].[] AS analyzed_table + WHERE (COALESCE(CAST([id] AS VARCHAR), CAST([created_at] AS VARCHAR)) IS NOT NULL) + GROUP BY[id], [created_at], analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ) grouping_table + GROUP BY analyzed_table.[country], analyzed_table.[state], DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1), DATEADD(month, DATEDIFF(month, 0, analyzed_table.[date_column]), 0) + ORDER BY level_1, level_2DATEFROMPARTS(YEAR(CAST(analyzed_table.[date_column] AS date)), MONTH(CAST(analyzed_table.[date_column] AS date)), 1) + + + ``` + ??? example "Trino" + + === "Sensor template for Trino" + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` + === "Rendered SQL for Trino" + ```sql + + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value, + + grouping_table.grouping_level_1, + + grouping_table.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY "id", "created_at") AS distinct_records, + + analyzed_table_nested.grouping_level_1, + + analyzed_table_nested.grouping_level_2 + , + time_period, + time_period_utc + FROM ( + SELECT + "id", "created_at", + analyzed_table_nested."country" AS grouping_level_1, + analyzed_table_nested."state" AS grouping_level_2, + DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS time_period, + CAST(DATE_TRUNC('MONTH', CAST(analyzed_table_nested."date_column" AS date)) AS TIMESTAMP) AS time_period_utc + FROM "your_trino_catalog".""."" AS analyzed_table_nested + WHERE (COALESCE(CAST("id" AS VARCHAR), CAST("created_at" AS VARCHAR)) IS NOT NULL) + ) + GROUP BY "id", "created_at", grouping_level_1, grouping_level_2, time_period, time_period_utc + ) grouping_table + GROUP BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ORDER BY grouping_level_1, grouping_level_2, time_period, time_period_utc + ``` + +___ + + + +## What's next +- Learn how to [configure data quality checks](../../../dqo-concepts/configuring-data-quality-checks-and-rules.md) in DQOps +- Look at the examples of [running data quality checks](../../../dqo-concepts/running-data-quality-checks.md), targeting tables and columns diff --git a/docs/checks/table/uniqueness/index.md b/docs/checks/table/uniqueness/index.md new file mode 100644 index 0000000000..f678d434b5 --- /dev/null +++ b/docs/checks/table/uniqueness/index.md @@ -0,0 +1,47 @@ +--- +title: table level uniqueness data quality checks +--- +# table level uniqueness data quality checks + +This is a list of uniqueness table data quality checks supported by DQOps and a brief description of what data quality issued they detect. + + + + +## table-level uniqueness checks + + +### [duplicate record count](./duplicate-record-count.md) +This check counts duplicate records values. It raises a data quality issue when the number of duplicates is above a minimum accepted value. + The default configuration detects duplicate rows by enforcing that the *min_count* of duplicates is zero. + + +| Data quality check name | Friendly name | Check type | Description | Standard | +|-------------------------|---------------|------------|-------------|----------| +|[`profile_duplicate_record_count`](./duplicate-record-count.md#profile-duplicate-record-count)|Maximum count of duplicate records|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|:material-check-bold:| +|[`daily_duplicate_record_count`](./duplicate-record-count.md#daily-duplicate-record-count)|Maximum count of duplicate records|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|:material-check-bold:| +|[`monthly_duplicate_record_count`](./duplicate-record-count.md#monthly-duplicate-record-count)|Maximum count of duplicate records|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|:material-check-bold:| +|[`daily_partition_duplicate_record_count`](./duplicate-record-count.md#daily-partition-duplicate-record-count)|Maximum count of duplicate records|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|:material-check-bold:| +|[`monthly_partition_duplicate_record_count`](./duplicate-record-count.md#monthly-partition-duplicate-record-count)|Maximum count of duplicate records|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|:material-check-bold:| + + + +### [duplicate record percent](./duplicate-record-percent.md) +This check measures the percentage of duplicate records values. It raises a data quality issue when the percentage of duplicates is above a minimum accepted value. + The default threshold is 0% duplicate values. + + +| Data quality check name | Friendly name | Check type | Description | Standard | +|-------------------------|---------------|------------|-------------|----------| +|[`profile_duplicate_record_percent`](./duplicate-record-percent.md#profile-duplicate-record-percent)|Maximum percentage of duplicate records|[profiling](../../../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|:material-check-bold:| +|[`daily_duplicate_record_percent`](./duplicate-record-percent.md#daily-duplicate-record-percent)|Maximum percentage of duplicate records|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|:material-check-bold:| +|[`monthly_duplicate_record_percent`](./duplicate-record-percent.md#monthly-duplicate-record-percent)|Maximum percentage of duplicate records|[monitoring](../../../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|:material-check-bold:| +|[`daily_partition_duplicate_record_percent`](./duplicate-record-percent.md#daily-partition-duplicate-record-percent)|Maximum percentage of duplicate records|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|:material-check-bold:| +|[`monthly_partition_duplicate_record_percent`](./duplicate-record-percent.md#monthly-partition-duplicate-record-percent)|Maximum percentage of duplicate records|[partitioned](../../../dqo-concepts/definition-of-data-quality-checks/partition-checks.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|:material-check-bold:| + + + + + + + diff --git a/docs/client/index.md b/docs/client/index.md index 3142614e04..39ea567fc3 100644 --- a/docs/client/index.md +++ b/docs/client/index.md @@ -422,7 +422,10 @@ ImportTablesQueueJobResult( ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) ] ), @@ -624,7 +627,8 @@ TableCurrentDataQualityStatusModel( }, dimensions={ - } + }, + table_exist=True ) ``` @@ -673,13 +677,14 @@ Are you looking to address a specific issue? Head down here for full reference d |[CheckResults](./operations/check_results.md)|Returns all the data quality check results of executed checks on tables and columns.| |[CheckResultsOverview](./operations/check_results_overview.md)|Returns the overview of the recently executed checks on tables and columns, returning a summary of the last 5 runs.| |[Checks](./operations/checks.md)|Data quality check definition management operations for adding/removing/changing custom data quality checks.| +|[ColumnQualityPolicies](./operations/column_quality_policies.md)|Operations for managing the configuration of data quality policies at a column level. Policies are the default configuration of data quality checks for columns matching a pattern.| |[Columns](./operations/columns.md)|Operations related to manage the metadata of columns, and managing the configuration of column-level data quality checks.| |[Connections](./operations/connections.md)|Operations for adding/updating/deleting the configuration of data sources managed by DQOps.| |[Dashboards](./operations/dashboards.md)|Operations for retrieving the list of data quality dashboards supported by DQOps and issuing short-term access keys to open a dashboard.| +|[DataDomains](./operations/data_domains.md)|Data domain management API to create different data domains.| |[DataGroupingConfigurations](./operations/data_grouping_configurations.md)|Operations for managing the configuration of data groupings on a table level in DQOps.| +|[DataLineage](./operations/data_lineage.md)|Operations related to managing and inspecting table and column lineage.| |[DataSources](./operations/data_sources.md)|Rest API controller that operates on data sources that are not yet imported, testing connections or retrieving the metadata (schemas and tables).| -|[DefaultColumnCheckPatterns](./operations/default_column_check_patterns.md)|Operations for managing the configuration of the default column-level checks for columns matching a pattern.| -|[DefaultTableCheckPatterns](./operations/default_table_check_patterns.md)|Operations for managing the configuration of the default table-level checks for tables matching a pattern.| |[Defaults](./operations/defaults.md)|Default settings management for configuring the default data quality checks that are configured for all imported tables and columns.| |[Dictionaries](./operations/dictionaries.md)|Operations for managing data dictionary CSV files in DQOps. Data dictionaries can be used in *accepted_values* data quality checks.| |[Environment](./operations/environment.md)|DQOps environment and configuration controller, provides access to the DQOps configuration, current user's information and issue local API Keys for the calling user.| @@ -700,6 +705,7 @@ Are you looking to address a specific issue? Head down here for full reference d |[SharedCredentials](./operations/shared_credentials.md)|Operations for managing shared credentials in DQOps. Credentials that are stored in the shared .credentials folder in the DQOps user's home folder.| |[TableComparisonResults](./operations/table_comparison_results.md)|Operations that returns the results of the most recent table comparison that was performed between the compared table and the reference table (the source of truth).| |[TableComparisons](./operations/table_comparisons.md)|Operations for managing the configurations of table comparisons between tables on the same or different data sources| +|[TableQualityPolicies](./operations/table_quality_policies.md)|Operations for managing the configuration of data quality policies at a table level. Policies are the default configuration of data quality checks for tables matching a pattern.| |[Tables](./operations/tables.md)|Operations related to manage the metadata of imported tables, and managing the configuration of table-level data quality checks.| |[Timezones](./operations/timezones.md)|Operations for returning time zone names and codes supported by DQOps.| |[Users](./operations/users.md)|Operations for managing access for DQOps users in a multi-user installations. User management is supported in the TEAM and ENTERPRISE licences.| diff --git a/docs/client/models/check_results.md b/docs/client/models/check_results.md index 7d72d0ce05..ac87a4b36d 100644 --- a/docs/client/models/check_results.md +++ b/docs/client/models/check_results.md @@ -50,6 +50,7 @@ The table validity status. It is a summary of the results of the most recently e |`checks`|The dictionary of statuses for data quality checks. The keys are data quality check names, the values are the current data quality check statuses that describe the most current status.|*Dict[string, [CheckCurrentDataQualityStatusModel](./common.md#checkcurrentdataqualitystatusmodel)]*| |`columns`|Dictionary of data statues for all columns that have any known data quality results. The keys in the dictionary are the column names.|*Dict[string, [ColumnCurrentDataQualityStatusModel](./common.md#columncurrentdataqualitystatusmodel)]*| |`dimensions`|Dictionary of the current data quality statues for each data quality dimension.|*Dict[string, [DimensionCurrentDataQualityStatusModel](./common.md#dimensioncurrentdataqualitystatusmodel)]*| +|`table_exist`|The flag informing whether the table exist. The table might not exist for the imported data lineage source tables.|*boolean*| ___ diff --git a/docs/client/models/column_quality_policies.md b/docs/client/models/column_quality_policies.md new file mode 100644 index 0000000000..91b8740cc0 --- /dev/null +++ b/docs/client/models/column_quality_policies.md @@ -0,0 +1,131 @@ +--- +title: DQOps REST API column_quality_policies models reference +--- +# DQOps REST API column_quality_policies models reference +The references of all objects used by [column_quality_policies](../operations/column_quality_policies.md) REST API operations are listed below. + + +## DataTypeCategory +Enumeration of common data type categories of data types. The providers will use this information to answer + which of their native data types matches a category. Some sensors (and profilers) cannot operate on some data types. + + +**The structure of this object is described below** + + +| Data type | Enum values | +|-----------|-------------| +|string|numeric_integer
numeric_decimal
numeric_float
datetime_timestamp
datetime_datetime
datetime_date
datetime_time
text
clob
json
bool
binary
array
other
| + +___ + +## TargetColumnPatternSpec +The configuration of a column pattern to match default column checks. Includes also the pattern for the target table. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`column`|The target column name filter. Accepts wildcards in the format: *id, *, c_*.|*string*| +|`data_type`|The target column data type filter. Filters by a physical (database specific) data type name imported from the data source. Accepts wildcards in the format: *int, *, big*.|*string*| +|[`data_type_category`](#datatypecategory)|The filter for a target data type category.|*[DataTypeCategory](#datatypecategory)*| +|`connection`|The data source connection name filter. Accepts wildcards in the format: *conn, *, conn*.|*string*| +|`schema`|The schema name filter. Accepts wildcards in the format: *_prod, *, pub*.|*string*| +|`table`|The table name filter. Accepts wildcards in the format: *_customers, *, fact_*.|*string*| +|`stage`|The table stage filter. Accepts wildcards in the format: *_landing, *, staging_*.|*string*| +|`table_priority`|The maximum table priority (inclusive) for tables that are covered by the default checks.|*integer*| +|`label`|The label filter. Accepts wildcards in the format: *_customers, *, fact_*. The label must be present on the connection or table.|*string*| + + +___ + +## ColumnQualityPolicyListModel +The listing model of column-level default check patterns that is returned by the REST API. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`policy_name`|Quality policy name.|*string*| +|`priority`|The priority of the policy. Policies with lower values are applied before policies with higher priority values.|*integer*| +|`disabled`|Disables this data quality check configuration. The checks will not be activated.|*boolean*| +|`description`|The description (documentation) of this data quality check configuration.|*string*| +|[`target_column`](#targetcolumnpatternspec)|The filters for the target column.|*[TargetColumnPatternSpec](#targetcolumnpatternspec)*| +|`can_edit`|Boolean flag that decides if the current user can update or delete this object.|*boolean*| +|`yaml_parsing_error`|Optional parsing error that was captured when parsing the YAML file. This field is null when the YAML file is valid. If an error was captured, this field returns the file parsing error message and the file location.|*string*| + + +___ + +## ColumnMonitoringCheckCategoriesSpec +Container of column level monitoring, divided by the time window (daily, monthly, etc.) + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`daily`](./columns.md#columndailymonitoringcheckcategoriesspec)|Configuration of daily monitoring evaluated at a column level.|*[ColumnDailyMonitoringCheckCategoriesSpec](./columns.md#columndailymonitoringcheckcategoriesspec)*| +|[`monthly`](./columns.md#columnmonthlymonitoringcheckcategoriesspec)|Configuration of monthly monitoring evaluated at a column level.|*[ColumnMonthlyMonitoringCheckCategoriesSpec](./columns.md#columnmonthlymonitoringcheckcategoriesspec)*| + + +___ + +## ColumnPartitionedCheckCategoriesSpec +Container of column level partitioned checks, divided by the time window (daily, monthly, etc.) + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`daily`](./columns.md#columndailypartitionedcheckcategoriesspec)|Configuration of day partitioned data quality checks evaluated at a column level.|*[ColumnDailyPartitionedCheckCategoriesSpec](./columns.md#columndailypartitionedcheckcategoriesspec)*| +|[`monthly`](./columns.md#columnmonthlypartitionedcheckcategoriesspec)|Configuration of monthly partitioned data quality checks evaluated at a column level.|*[ColumnMonthlyPartitionedCheckCategoriesSpec](./columns.md#columnmonthlypartitionedcheckcategoriesspec)*| + + +___ + +## ColumnQualityPolicySpec +The default configuration of column-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on columns. + This configuration serves as a data quality policy that defines the data quality checks that are verified on matching columns. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`priority`|The priority of the pattern. Patterns with lower values are applied before patterns with higher priority values.|*integer*| +|`disabled`|Disables this data quality check configuration. The checks will not be activated.|*boolean*| +|`description`|The description (documentation) of this data quality check configuration.|*string*| +|[`target`](./column_quality_policies.md#targetcolumnpatternspec)|The target column filter that are filtering the column, table and connection on which the default checks are applied.|*[TargetColumnPatternSpec](./column_quality_policies.md#targetcolumnpatternspec)*| +|[`profiling_checks`](./columns.md#columnprofilingcheckcategoriesspec)|Configuration of data quality profiling checks that are enabled. Pick a check from a category, apply the parameters and rules to enable it.|*[ColumnProfilingCheckCategoriesSpec](./columns.md#columnprofilingcheckcategoriesspec)*| +|[`monitoring_checks`](#columnmonitoringcheckcategoriesspec)|Configuration of table level monitoring checks. Monitoring checks are data quality checks that are evaluated for each period of time (daily, weekly, monthly, etc.). A monitoring check stores only the most recent data quality check result for each period of time.|*[ColumnMonitoringCheckCategoriesSpec](#columnmonitoringcheckcategoriesspec)*| +|[`partitioned_checks`](#columnpartitionedcheckcategoriesspec)|Configuration of table level date/time partitioned checks. Partitioned data quality checks are evaluated for each partition separately, raising separate alerts at a partition level. The table does not need to be physically partitioned by date, it is possible to run data quality checks for each day or month of data separately.|*[ColumnPartitionedCheckCategoriesSpec](#columnpartitionedcheckcategoriesspec)*| + + +___ + +## ColumnQualityPolicyModel +Default column-level checks pattern model that is returned by the REST API. Describes a configuration of data quality checks for a named pattern. DQOps applies these checks on columns that match the filter. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`policy_name`|Quality policy name|*string*| +|[`policy_spec`](#columnqualitypolicyspec)|The quality policy specification.|*[ColumnQualityPolicySpec](#columnqualitypolicyspec)*| +|`can_edit`|Boolean flag that decides if the current user can update or delete this object.|*boolean*| +|`yaml_parsing_error`|Optional parsing error that was captured when parsing the YAML file. This field is null when the YAML file is valid. If an error was captured, this field returns the file parsing error message and the file location.|*string*| + + +___ + diff --git a/docs/client/models/common.md b/docs/client/models/common.md index 4f0b3db657..f71944dd6d 100644 --- a/docs/client/models/common.md +++ b/docs/client/models/common.md @@ -597,6 +597,7 @@ Column list model that returns the basic fields from a column specification, exc |[`run_partition_checks_job_template`](./common.md#checksearchfilters)|Configured parameters for the "check run" job that should be pushed to the job queue in order to run partition partitioned checks within this column.|*[CheckSearchFilters](./common.md#checksearchfilters)*| |[`collect_statistics_job_template`](./jobs.md#statisticscollectorsearchfilters)|Configured parameters for the "collect statistics" job that should be pushed to the job queue in order to run all statistics collector within this column.|*[StatisticsCollectorSearchFilters](./jobs.md#statisticscollectorsearchfilters)*| |[`data_clean_job_template`](./jobs.md#deletestoreddataqueuejobparameters)|Configured parameters for the "data clean" job that after being supplied with a time range should be pushed to the job queue in order to remove stored results connected with this column.|*[DeleteStoredDataQueueJobParameters](./jobs.md#deletestoreddataqueuejobparameters)*| +|`advanced_properties`|A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value dictionary.|*Dict[string, string]*| |`can_edit`|Boolean flag that decides if the current user can update or delete the column.|*boolean*| |`can_collect_statistics`|Boolean flag that decides if the current user can collect statistics.|*boolean*| |`can_run_checks`|Boolean flag that decides if the current user can run checks.|*boolean*| @@ -650,6 +651,7 @@ Connection model returned by the rest api that is limited only to the basic fiel |[`run_partition_checks_job_template`](./common.md#checksearchfilters)|Configured parameters for the "check run" job that should be pushed to the job queue in order to run partition partitioned checks within this connection.|*[CheckSearchFilters](./common.md#checksearchfilters)*| |[`collect_statistics_job_template`](./jobs.md#statisticscollectorsearchfilters)|Configured parameters for the "collect statistics" job that should be pushed to the job queue in order to run all statistics collectors within this connection.|*[StatisticsCollectorSearchFilters](./jobs.md#statisticscollectorsearchfilters)*| |[`data_clean_job_template`](./jobs.md#deletestoreddataqueuejobparameters)|Configured parameters for the "data clean" job that after being supplied with a time range should be pushed to the job queue in order to remove stored results connected with this connection.|*[DeleteStoredDataQueueJobParameters](./jobs.md#deletestoreddataqueuejobparameters)*| +|`advanced_properties`|A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value dictionary.|*Dict[string, string]*| |`can_edit`|Boolean flag that decides if the current user can update or delete the connection to the data source.|*boolean*| |`can_collect_statistics`|Boolean flag that decides if the current user can collect statistics.|*boolean*| |`can_run_checks`|Boolean flag that decides if the current user can run checks.|*boolean*| @@ -723,6 +725,7 @@ Table list model returned by the rest api that is limited only to the basic fiel |[`run_partition_checks_job_template`](./common.md#checksearchfilters)|Configured parameters for the "check run" job that should be pushed to the job queue in order to run partition partitioned checks within this table.|*[CheckSearchFilters](./common.md#checksearchfilters)*| |[`collect_statistics_job_template`](./jobs.md#statisticscollectorsearchfilters)|Configured parameters for the "collect statistics" job that should be pushed to the job queue in order to run all statistics collectors within this table.|*[StatisticsCollectorSearchFilters](./jobs.md#statisticscollectorsearchfilters)*| |[`data_clean_job_template`](./jobs.md#deletestoreddataqueuejobparameters)|Configured parameters for the "data clean" job that after being supplied with a time range should be pushed to the job queue in order to remove stored results connected with this table.|*[DeleteStoredDataQueueJobParameters](./jobs.md#deletestoreddataqueuejobparameters)*| +|`advanced_properties`|A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value dictionary.|*Dict[string, string]*| |`can_edit`|Boolean flag that decides if the current user can update or delete this object.|*boolean*| |`can_collect_statistics`|Boolean flag that decides if the current user can collect statistics.|*boolean*| |`can_run_checks`|Boolean flag that decides if the current user can run checks.|*boolean*| diff --git a/docs/client/models/data_domains.md b/docs/client/models/data_domains.md new file mode 100644 index 0000000000..9a9c929bed --- /dev/null +++ b/docs/client/models/data_domains.md @@ -0,0 +1,23 @@ +--- +title: DQOps REST API data_domains models reference +--- +# DQOps REST API data_domains models reference +The references of all objects used by [data_domains](../operations/data_domains.md) REST API operations are listed below. + + +## LocalDataDomainModel +Model that describes a local data domain. It is also used to create new data domains. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`domain_name`|Data domain name.|*string*| +|`display_name`|Data domain display name (user-friendly name).|*string*| +|`enable_scheduler`|Enables the job scheduler for this domain.|*boolean*| + + +___ + diff --git a/docs/client/models/data_lineage.md b/docs/client/models/data_lineage.md new file mode 100644 index 0000000000..e7740634ea --- /dev/null +++ b/docs/client/models/data_lineage.md @@ -0,0 +1,92 @@ +--- +title: DQOps REST API data_lineage models reference +--- +# DQOps REST API data_lineage models reference +The references of all objects used by [data_lineage](../operations/data_lineage.md) REST API operations are listed below. + + +## TableLineageSourceListModel +Data lineage model that describes one source table of the current table. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`target_connection`|The connection name where the target table is defined.|*string*| +|`target_schema`|The schema name in the target connection where the target table is defined.|*string*| +|`target_table`|The name of the target table inside the target schema.|*string*| +|`source_connection`|The name of a source connection that is defined in DQOps and contains a source table from which the current table receives data.|*string*| +|`source_schema`|The name of a source schema within the source connection that contains a source table from which the current table receives data.|*string*| +|`source_table`|The name of a source schema within the source connection that contains a source table from which the current table receives data.|*string*| +|`data_lineage_source_tool`|The name of a source tool from which this data lineage information was copied. This field should be filled when the data lineage was imported from another data catalog or a data lineage tracking platform.|*string*| +|`properties`|A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external data lineage sources can use it to store mapping information.|*Dict[string, string]*| +|`can_edit`|Boolean flag that decides if the current user can update or delete this object.|*boolean*| +|[`source_table_data_quality_status`](./check_results.md#tablecurrentdataqualitystatusmodel)|The current data quality status for the table, grouped by data quality dimensions. DQOps may return a null value when the results were not yet loaded into the cache. In that case, the client should wait a few seconds and retry a call to get the most recent data quality status of the table.|*[TableCurrentDataQualityStatusModel](./check_results.md#tablecurrentdataqualitystatusmodel)*| + + +___ + +## SourceColumnsSetSpec +A collection of unique names of source columns from which the current column receives data. This information is used to track column-level data lineage. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| + + +___ + +## ColumnLineageSourceSpec +Describes the list of source columns for a column in the current table. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`source_columns`](#sourcecolumnssetspec)|A list of source columns from the source table name from which this column receives data.|*[SourceColumnsSetSpec](#sourcecolumnssetspec)*| +|`properties`|A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external data lineage sources can use it to store mapping information.|*Dict[string, string]*| + + +___ + +## ColumnLineageSourceSpecMap +Dictionary of mapping of source columns to the columns in the current table. + The keys in this dictionary are the column names in the current table. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`self`||*Dict[string, [ColumnLineageSourceSpec](./data_lineage.md#columnlineagesourcespec)]*| + + +___ + +## TableLineageSourceSpec +Data lineage specification for a table to identify a source table of the current table where this object is stored. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`source_connection`|The name of a source connection that is defined in DQOps and contains a source table from which the current table receives data.|*string*| +|`source_schema`|The name of a source schema within the source connection that contains a source table from which the current table receives data.|*string*| +|`source_table`|The name of a source table in the source schema from which the current table receives data.|*string*| +|`data_lineage_source_tool`|The name of a source tool from which this data lineage information was copied. This field should be filled when the data lineage was imported from another data catalog or a data lineage tracking platform.|*string*| +|`properties`|A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external data lineage sources can use it to store mapping information.|*Dict[string, string]*| +|[`columns`](#columnlineagesourcespecmap)|Configuration of source columns for each column in the current table. The keys in this dictionary are column names in the current table. The object stored in the dictionary contain a list of source columns.|*[ColumnLineageSourceSpecMap](#columnlineagesourcespecmap)*| + + +___ + diff --git a/docs/client/models/environment.md b/docs/client/models/environment.md index 43ddc6fdac..57ebfbe333 100644 --- a/docs/client/models/environment.md +++ b/docs/client/models/environment.md @@ -69,6 +69,7 @@ The model that describes the current user and his access rights. |`can_manage_users`|User can manage other users, add users to a multi-user account, change access rights, reset passwords.|*boolean*| |`can_manage_and_view_shared_credentials`|User can manage shared credentials and view (or download) already defined shared credentials.|*boolean*| |`can_change_own_password`|User can change his own password in DQOps Cloud, because the DQOps Cloud Pairing API Key is valid and synchronization is enabled.|*boolean*| +|`can_use_data_domains`|User can use data domains. Support for data domains requires an ENTERPRISE license of DQOps.|*boolean*| ___ diff --git a/docs/client/models/index.md b/docs/client/models/index.md index 6f54805168..214f1636c9 100644 --- a/docs/client/models/index.md +++ b/docs/client/models/index.md @@ -73,6 +73,19 @@ This is a list of the models in DQOps REST API Python client broken down by indi |[*CheckDefinitionModel*](./checks.md#checkdefinitionmodel)|Check model that is returned by the REST API. Describes a single unique data quality check.| +## column_quality_policies + +| Class name | Description                     | +|------------|---------------------------------| +|[*DataTypeCategory*](./column_quality_policies.md#datatypecategory)|Enumeration of common data type categories of data types. The providers will use this information to answer which of their native data types matches a category. Some sensors (and profilers) cannot operate on some data types.| +|[*TargetColumnPatternSpec*](./column_quality_policies.md#targetcolumnpatternspec)|The configuration of a column pattern to match default column checks. Includes also the pattern for the target table.| +|[*ColumnQualityPolicyListModel*](./column_quality_policies.md#columnqualitypolicylistmodel)|The listing model of column-level default check patterns that is returned by the REST API.| +|[*ColumnMonitoringCheckCategoriesSpec*](./column_quality_policies.md#columnmonitoringcheckcategoriesspec)|Container of column level monitoring, divided by the time window (daily, monthly, etc.)| +|[*ColumnPartitionedCheckCategoriesSpec*](./column_quality_policies.md#columnpartitionedcheckcategoriesspec)|Container of column level partitioned checks, divided by the time window (daily, monthly, etc.)| +|[*ColumnQualityPolicySpec*](./column_quality_policies.md#columnqualitypolicyspec)|The default configuration of column-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on columns. This configuration serves as a data quality policy that defines the data quality checks that are verified on matching columns.| +|[*ColumnQualityPolicyModel*](./column_quality_policies.md#columnqualitypolicymodel)|Default column-level checks pattern model that is returned by the REST API. Describes a configuration of data quality checks for a named pattern. DQOps applies these checks on columns that match the filter.| + + ## columns | Class name | Description                     | @@ -113,48 +126,41 @@ This is a list of the models in DQOps REST API Python client broken down by indi |[*DashboardsFolderSpec*](./dashboards.md#dashboardsfolderspec)|Description of a folder with multiple dashboards or other folders.| -## data_grouping_configurations +## data_domains | Class name | Description                     | |------------|---------------------------------| -|[*DataGroupingConfigurationListModel*](./data_grouping_configurations.md#datagroupingconfigurationlistmodel)|Basic model for data grouping configuration on a table, returned by the rest api.| -|[*DataGroupingConfigurationModel*](./data_grouping_configurations.md#datagroupingconfigurationmodel)|Model of data grouping configuration on a table returned by the rest api, including all configuration information.| -|[*DataGroupingConfigurationTrimmedModel*](./data_grouping_configurations.md#datagroupingconfigurationtrimmedmodel)|Data grouping on a table model with trimmed access path.| +|[*LocalDataDomainModel*](./data_domains.md#localdatadomainmodel)|Model that describes a local data domain. It is also used to create new data domains.| -## data_sources +## data_grouping_configurations | Class name | Description                     | |------------|---------------------------------| -|[*ConnectionTestStatus*](./data_sources.md#connectionteststatus)|Tabular output format for printing the tabular results.| -|[*ConnectionTestModel*](./data_sources.md#connectiontestmodel)|Connection test status result model returned from REST API. Describes the status of testing a connection (opening a connection to verify if it usable, credentials are approved and the access was granted by the tested data source).| -|[*RemoteTableListModel*](./data_sources.md#remotetablelistmodel)|Remote table list model that is returned when a data source is introspected to retrieve the list of tables available in a data source.| -|[*SchemaRemoteModel*](./data_sources.md#schemaremotemodel)|Schema model returned from REST API. Describes a schema on the source database with established connection.| +|[*DataGroupingConfigurationListModel*](./data_grouping_configurations.md#datagroupingconfigurationlistmodel)|Basic model for data grouping configuration on a table, returned by the rest api.| +|[*DataGroupingConfigurationModel*](./data_grouping_configurations.md#datagroupingconfigurationmodel)|Model of data grouping configuration on a table returned by the rest api, including all configuration information.| +|[*DataGroupingConfigurationTrimmedModel*](./data_grouping_configurations.md#datagroupingconfigurationtrimmedmodel)|Data grouping on a table model with trimmed access path.| -## default_column_check_patterns +## data_lineage | Class name | Description                     | |------------|---------------------------------| -|[*DataTypeCategory*](./default_column_check_patterns.md#datatypecategory)|Enumeration of common data type categories of data types. The providers will use this information to answer which of their native data types matches a category. Some sensors (and profilers) cannot operate on some data types.| -|[*TargetColumnPatternSpec*](./default_column_check_patterns.md#targetcolumnpatternspec)|The configuration of a column pattern to match default column checks. Includes also the pattern for the target table.| -|[*DefaultColumnChecksPatternListModel*](./default_column_check_patterns.md#defaultcolumncheckspatternlistmodel)|The listing model of column-level default check patterns that is returned by the REST API.| -|[*ColumnMonitoringCheckCategoriesSpec*](./default_column_check_patterns.md#columnmonitoringcheckcategoriesspec)|Container of column level monitoring, divided by the time window (daily, monthly, etc.)| -|[*ColumnPartitionedCheckCategoriesSpec*](./default_column_check_patterns.md#columnpartitionedcheckcategoriesspec)|Container of column level partitioned checks, divided by the time window (daily, monthly, etc.)| -|[*ColumnDefaultChecksPatternSpec*](./default_column_check_patterns.md#columndefaultcheckspatternspec)|The default configuration of column-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on columns. This configuration serves as a data quality policy that defines the data quality checks that are verified on matching columns.| -|[*DefaultColumnChecksPatternModel*](./default_column_check_patterns.md#defaultcolumncheckspatternmodel)|Default column-level checks pattern model that is returned by the REST API. Describes a configuration of data quality checks for a named pattern. DQOps applies these checks on columns that match the filter.| +|[*TableLineageSourceListModel*](./data_lineage.md#tablelineagesourcelistmodel)|Data lineage model that describes one source table of the current table.| +|[*SourceColumnsSetSpec*](./data_lineage.md#sourcecolumnssetspec)|A collection of unique names of source columns from which the current column receives data. This information is used to track column-level data lineage.| +|[*ColumnLineageSourceSpec*](./data_lineage.md#columnlineagesourcespec)|Describes the list of source columns for a column in the current table.| +|[*ColumnLineageSourceSpecMap*](./data_lineage.md#columnlineagesourcespecmap)|Dictionary of mapping of source columns to the columns in the current table. The keys in this dictionary are the column names in the current table.| +|[*TableLineageSourceSpec*](./data_lineage.md#tablelineagesourcespec)|Data lineage specification for a table to identify a source table of the current table where this object is stored.| -## default_table_check_patterns +## data_sources | Class name | Description                     | |------------|---------------------------------| -|[*TargetTablePatternSpec*](./default_table_check_patterns.md#targettablepatternspec)|The configuration of a table pattern to match default table checks.| -|[*DefaultTableChecksPatternListModel*](./default_table_check_patterns.md#defaulttablecheckspatternlistmodel)|The listing model of table-level default check patterns that is returned by the REST API.| -|[*TableMonitoringCheckCategoriesSpec*](./default_table_check_patterns.md#tablemonitoringcheckcategoriesspec)|Container of table level monitoring, divided by the time window (daily, monthly, etc.)| -|[*TablePartitionedCheckCategoriesSpec*](./default_table_check_patterns.md#tablepartitionedcheckcategoriesspec)|Container of table level partitioned checks, divided by the time window (daily, monthly, etc.)| -|[*TableDefaultChecksPatternSpec*](./default_table_check_patterns.md#tabledefaultcheckspatternspec)|The default configuration of table-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on tables. This configuration serves as a data quality policy that defines the data quality checks that are verified on matching tables.| -|[*DefaultTableChecksPatternModel*](./default_table_check_patterns.md#defaulttablecheckspatternmodel)|Default table-level checks pattern model that is returned by the REST API. Describes a configuration of data quality checks for a named pattern. DQOps applies these checks on tables that match the filter.| +|[*ConnectionTestStatus*](./data_sources.md#connectionteststatus)|Tabular output format for printing the tabular results.| +|[*ConnectionTestModel*](./data_sources.md#connectiontestmodel)|Connection test status result model returned from REST API. Describes the status of testing a connection (opening a connection to verify if it usable, credentials are approved and the access was granted by the tested data source).| +|[*RemoteTableListModel*](./data_sources.md#remotetablelistmodel)|Remote table list model that is returned when a data source is introspected to retrieve the list of tables available in a data source.| +|[*SchemaRemoteModel*](./data_sources.md#schemaremotemodel)|Schema model returned from REST API. Describes a schema on the source database with established connection.| ## dictionaries @@ -359,6 +365,18 @@ This is a list of the models in DQOps REST API Python client broken down by indi |[*TableComparisonModel*](./table_comparisons.md#tablecomparisonmodel)|Model that contains the all editable information about a table-to-table comparison defined on a compared table.| +## table_quality_policies + +| Class name | Description                     | +|------------|---------------------------------| +|[*TargetTablePatternSpec*](./table_quality_policies.md#targettablepatternspec)|The configuration of a table pattern to match default table checks.| +|[*TableQualityPolicyListModel*](./table_quality_policies.md#tablequalitypolicylistmodel)|The listing model of table-level default check patterns that is returned by the REST API.| +|[*TableMonitoringCheckCategoriesSpec*](./table_quality_policies.md#tablemonitoringcheckcategoriesspec)|Container of table level monitoring, divided by the time window (daily, monthly, etc.)| +|[*TablePartitionedCheckCategoriesSpec*](./table_quality_policies.md#tablepartitionedcheckcategoriesspec)|Container of table level partitioned checks, divided by the time window (daily, monthly, etc.)| +|[*TableQualityPolicySpec*](./table_quality_policies.md#tablequalitypolicyspec)|The default configuration of table-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on tables. This configuration serves as a data quality policy that defines the data quality checks that are verified on matching tables.| +|[*TableQualityPolicyModel*](./table_quality_policies.md#tablequalitypolicymodel)|Default table-level checks pattern model that is returned by the REST API. Describes a configuration of data quality checks for a named pattern. DQOps applies these checks on tables that match the filter.| + + ## tables | Class name | Description                     | @@ -384,6 +402,6 @@ This is a list of the models in DQOps REST API Python client broken down by indi | Class name | Description                     | |------------|---------------------------------| -|[*DqoCloudUserModel*](./users.md#dqocloudusermodel)|DQOps Cloud user model - identifies a user in a multi-user DQOps deployment.| +|[*DqoUserRolesModel*](./users.md#dqouserrolesmodel)|DQOps user model - identifies a user in a multi-user DQOps deployment and the user's roles.| diff --git a/docs/client/models/jobs.md b/docs/client/models/jobs.md index 67c6f55c62..96e922a6a6 100644 --- a/docs/client/models/jobs.md +++ b/docs/client/models/jobs.md @@ -22,6 +22,7 @@ The references of all objects used by [jobs](../operations/jobs.md) REST API ope |`from_date_time`|Analyze the data since the given date and time (inclusive). The date and time should be an ISO 8601 local date and time without the time zone (yyyy-MM-dd HH\:mm:ss). The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the beginning date and time overrides recent days and recent months.|*datetime*| |`to_date`|Analyze the data until the given date (exclusive, the given date and the following dates are not analyzed). The date should be an ISO 8601 date (YYYY-MM-DD). The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date overrides the parameters to disable analyzing today or the current month.|*date*| |`to_date_time`|Analyze the data until the given date and time (exclusive). The date should be an ISO 8601 date (yyyy-MM-dd). The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date and time overrides the parameters to disable analyzing today or the current month.|*datetime*| +|`where_filter`|An additional filter which must be a valid SQL predicate (an SQL expression that returns 'true' or 'false') that is added to the WHERE clause of the SQL query that DQOps will run on the data source. The purpose of a custom filter is to analyze only a subset of data, for example, when a new batch of records is loaded, and the data quality checks are evaluated as a data contract. All the records in that batch must tagged with the same value, and the passed predicate to find records from that batch would use the filter in the form: "{alias}.batch_id = 1". The filter can use replacement tokens {alias} to reference the analyzed table.|*string*| ___ @@ -156,7 +157,8 @@ Parameters for the "delete stored data* queue job that deletes data from pa |`delete_check_results`|Delete the data from the [check_results](../../reference/parquetfiles/check_results.md) table. Because the default value is *false*, this parameter must be set to *true* to delete the check results.|*boolean*| |`delete_sensor_readouts`|Delete the data from the [sensor_readouts](../../reference/parquetfiles/sensor_readouts.md) table. Because the default value is *false*, this parameter must be set to *true* to delete the sensor readouts.|*boolean*| |`delete_error_samples`|Delete the data from the [error_samples](../../reference/parquetfiles/error_samples.md) table. Because the default value is *false*, this parameter must be set to *true* to delete the error samples.|*boolean*| -|`delete_incidents`|Delete the data from the [incidents](../../reference/parquetfiles/incidents.md) table. Because the default value is *false*, this parameter must be set to *true* to delete the error samples.|*boolean*| +|`delete_incidents`|Delete the data from the [incidents](../../reference/parquetfiles/incidents.md) table. Because the default value is *false*, this parameter must be set to *true* to delete the incidents.|*boolean*| +|`delete_checks_configuration`|Delete the data quality configured checks from the table. They are detached from the configuration.Because the default value is *false*, this parameter must be set to *true* to delete the checks configuration.|*boolean*| |`column_names`|The list of column names to delete the data for column level results or errors only for selected columns.|*List[string]*| |`check_category`|The check category name, for example *volume* or *anomaly*.|*string*| |`table_comparison_name`|The name of a table comparison configuration. Deletes only table comparison results (and errors) for a given comparison.|*string*| @@ -424,7 +426,6 @@ Hierarchy node search filters for finding enabled statistics collectors (basic p |`sensor_name`|The target sensor name to run only data quality checks that are using this sensor. Uses the full sensor name which is the full folder path within the *sensors* folder. This field supports search patterns such as: 'table/volume/row_\*', '\*_count', 'table/volume/prefix_\*_suffix'.|*string*| |`collector_category`|The target statistics collector category, for example: *nulls*, *volume*, *sampling*.|*string*| |[`target`](#statisticscollectortarget)|The target type of object to collect statistics from. Supported values are: *table* to collect only table level statistics or *column* to collect only column level statistics.|*[StatisticsCollectorTarget](#statisticscollectortarget)*| -|`samples_limit`|The default limit of column samples that are collected.|*integer*| |`connection`|The connection (data source) name. Supports search patterns in the format: 'source\*', '\*_prod', 'prefix\*suffix'.|*string*| |`full_table_name`|The schema and table name. It is provided as *.*, for example *public.fact_sales*. The schema and table name accept patterns both in the schema name and table name parts. Sample patterns are: 'schema_name.tab_prefix_\*', 'schema_name.*', '*.*', 'schema_name.\*_customer', 'schema_name.tab_\*_suffix'.|*string*| |`enabled`|A boolean flag to target enabled tables, columns or checks. When the value of this field is not set, the default value of this field is *true*, targeting only tables, columns and checks that are not implicitly disabled.|*boolean*| @@ -457,6 +458,8 @@ ___ |---------------|---------------------------------|-----------| |[`statistics_collector_search_filters`](./jobs.md#statisticscollectorsearchfilters)|Statistics collectors search filters that identify the type of statistics collector to run.|*[StatisticsCollectorSearchFilters](./jobs.md#statisticscollectorsearchfilters)*| |[`data_scope`](#statisticsdatascope)|The target scope of collecting statistics. Statistics can be collected for the entire table or for each data grouping separately.|*[StatisticsDataScope](#statisticsdatascope)*| +|`configure_table`|Turns on a special mode of collecting statistics that will configure the timestamp and ID columns. It should be used only during the first statistics collection.|*boolean*| +|`samples_limit`|The default limit of column samples that are collected.|*integer*| |`dummy_sensor_execution`|Boolean flag that enables a dummy statistics collection (sensors are executed, but the statistics results are not written to the parquet files).|*boolean*| |[`collect_statistics_result`](./jobs.md#collectstatisticsresult)|The summary of the statistics collection job after if finished. Returns the number of collectors analyzed, columns analyzed, statistics results captured.|*[CollectStatisticsResult](./jobs.md#collectstatisticsresult)*| @@ -477,6 +480,8 @@ ___ |[`table`](./common.md#physicaltablename)|The full physical name (schema.table) of the target table.|*[PhysicalTableName](./common.md#physicaltablename)*| |[`statistics_collector_search_filters`](./jobs.md#statisticscollectorsearchfilters)|Statistics collectors search filters that identify the type of statistics collector to run.|*[StatisticsCollectorSearchFilters](./jobs.md#statisticscollectorsearchfilters)*| |[`data_scope`](./jobs.md#statisticsdatascope)|The target scope of collecting statistics. Statistics can be collected for the entire table or for each data grouping separately.|*[StatisticsDataScope](./jobs.md#statisticsdatascope)*| +|`samples_limit`|The default limit of column samples that are collected.|*integer*| +|`configure_table`|Turns on a special mode of collecting statistics that will configure the timestamp and ID columns. It should be used only during the first statistics collection.|*boolean*| |`dummy_sensor_execution`|Boolean flag that enables a dummy statistics collection (sensors are executed, but the statistics results are not written to the parquet files).|*boolean*| |[`collect_statistics_result`](./jobs.md#collectstatisticsresult)|The summary of the statistics collection job after if finished. Returns the number of collectors analyzed, columns analyzed, statistics results captured.|*[CollectStatisticsResult](./jobs.md#collectstatisticsresult)*| @@ -596,6 +601,7 @@ Model of a single job that was scheduled or has finished. It is stored in the jo |[`parameters`](#dqojobentryparametersmodel)||*[DqoJobEntryParametersModel](#dqojobentryparametersmodel)*| |[`status`](./jobs.md#dqojobstatus)||*[DqoJobStatus](./jobs.md#dqojobstatus)*| |`error_message`||*string*| +|`data_domain`||*string*| ___ @@ -613,6 +619,7 @@ Describes a change to the job status or the job queue (such as a new job was add |[`job_id`](./common.md#dqoqueuejobid)||*[DqoQueueJobId](./common.md#dqoqueuejobid)*| |`change_sequence`||*long*| |[`updated_model`](./jobs.md#dqojobhistoryentrymodel)||*[DqoJobHistoryEntryModel](./jobs.md#dqojobhistoryentrymodel)*| +|`domain_name`||*string*| ___ diff --git a/docs/client/models/rule_mining.md b/docs/client/models/rule_mining.md index 9bcec119aa..0813895767 100644 --- a/docs/client/models/rule_mining.md +++ b/docs/client/models/rule_mining.md @@ -34,7 +34,8 @@ Data quality check rule mining parameters. Configure what type of checks should |`copy_failed_profiling_checks`|Copy also the configuration of profiling checks that failed.|*boolean*| |`copy_disabled_profiling_checks`|Copy also the configuration of profiling checks that are disabled.|*boolean*| |`copy_profiling_checks`|Copy the configuration of valid profiling checks.|*boolean*| -|`propose_default_checks`|Propose the rules for default checks that were activated using data quality check patterns (policies). The default value of this parameter is 'true'.|*boolean*| +|`reconfigure_policy_enabled_checks`|Propose the rules for default checks that were activated using data quality check patterns (policies). The default value of this parameter is 'true'.|*boolean*| +|`propose_checks_from_statistics`|Propose the configuration of data quality checks from statistics.|*boolean*| |`propose_minimum_row_count`|Propose the default configuration of the minimum row count for monitoring checks (full table scans). The default value of this parameter is 'true'.|*boolean*| |`propose_column_count`|Propose the default configuration of the column count check. The default value of this parameter is 'true'.|*boolean*| |`propose_timeliness_checks`|Propose the default configuration of the timeliness checks, including an accepted freshness delay. The default value of this parameter is 'true'.|*boolean*| @@ -56,7 +57,7 @@ Data quality check rule mining parameters. Configure what type of checks should |`propose_standard_pattern_checks`|Propose the default configuration for the patterns check that validate the format of popular text patterns, such as UUIDs, phone numbers, or emails. DQOps will configure these data quality checks when analyzed columns contain enough values matching a standard pattern. The default value of this parameter is 'true'.|*boolean*| |`detect_regular_expressions`|Analyze sample text values and try to find a regular expression that detects valid values similar to the sample values. The default value of this parameter is 'true'.|*boolean*| |`propose_whitespace_checks`|Propose the default configuration for the whitespace detection checks. Whitespace checks detect common data quality issues with storing text representations of null values, such as 'null', 'None', 'n/a' and other texts that should be stored as regular NULL values. The default value of this parameter is 'true'.|*boolean*| -|`apply_pii_checks`|Apply the rules to the Personal Identifiable Information checks (sensitive data), but only when the checks were run as profiling checks activated manually, or by activating a data quality check pattern that scans all columns for PII data. The default value of this parameter is 'true'.|*boolean*| +|`apply_pii_checks`|Applies rules to Personal Identifiable Information checks (sensitive data), but only when the checks were activated manually as profiling checks, or through a data quality check pattern that scans all columns for PII data. The default value of this parameter is 'true'.|*boolean*| |`propose_custom_checks`|Propose the default configuration for custom checks that use built-in data quality rules. The default value of this parameter is 'true'.|*boolean*| |`fail_checks_at_percent_error_rows`|The percentage value captured by a profiling check (for example 0.03% of errors or 99.97% of valid) that is used to propose a percentage rule that will treat the values as errors (i.e., max_percent = 0%, or min_percent = 100%).|*double*| |`max_percent_error_rows_for_percent_checks`|The default maximum percentage of invalid rows for which the rule engine should configure rule values, especially min_percent, min_count or max_percent.|*double*| diff --git a/docs/client/models/table_quality_policies.md b/docs/client/models/table_quality_policies.md new file mode 100644 index 0000000000..158cae62d4 --- /dev/null +++ b/docs/client/models/table_quality_policies.md @@ -0,0 +1,114 @@ +--- +title: DQOps REST API table_quality_policies models reference +--- +# DQOps REST API table_quality_policies models reference +The references of all objects used by [table_quality_policies](../operations/table_quality_policies.md) REST API operations are listed below. + + +## TargetTablePatternSpec +The configuration of a table pattern to match default table checks. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`connection`|The data source connection name filter. Accepts wildcards in the format: *conn, *, conn*.|*string*| +|`schema`|The schema name filter. Accepts wildcards in the format: *_prod, *, pub*.|*string*| +|`table`|The table name filter. Accepts wildcards in the format: *_customers, *, fact_*.|*string*| +|`stage`|The table stage filter. Accepts wildcards in the format: *_landing, *, staging_*.|*string*| +|`table_priority`|The maximum table priority (inclusive) for tables that are covered by the default checks.|*integer*| +|`label`|The label filter. Accepts wildcards in the format: *_customers, *, fact_*. The label must be present on the connection or table.|*string*| + + +___ + +## TableQualityPolicyListModel +The listing model of table-level default check patterns that is returned by the REST API. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`policy_name`|Quality policy name.|*string*| +|`priority`|The priority of the policy. Policies with lower values are applied before policies with higher priority values.|*integer*| +|`disabled`|Disables this data quality check configuration. The checks will not be activated.|*boolean*| +|`description`|The description (documentation) of this data quality check configuration.|*string*| +|[`target_table`](#targettablepatternspec)|The filters for the target table.|*[TargetTablePatternSpec](#targettablepatternspec)*| +|`can_edit`|Boolean flag that decides if the current user can update or delete this object.|*boolean*| +|`yaml_parsing_error`|Optional parsing error that was captured when parsing the YAML file. This field is null when the YAML file is valid. If an error was captured, this field returns the file parsing error message and the file location.|*string*| + + +___ + +## TableMonitoringCheckCategoriesSpec +Container of table level monitoring, divided by the time window (daily, monthly, etc.) + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`daily`](./tables.md#tabledailymonitoringcheckcategoriesspec)|Configuration of daily monitoring evaluated at a table level.|*[TableDailyMonitoringCheckCategoriesSpec](./tables.md#tabledailymonitoringcheckcategoriesspec)*| +|[`monthly`](./tables.md#tablemonthlymonitoringcheckcategoriesspec)|Configuration of monthly monitoring evaluated at a table level.|*[TableMonthlyMonitoringCheckCategoriesSpec](./tables.md#tablemonthlymonitoringcheckcategoriesspec)*| + + +___ + +## TablePartitionedCheckCategoriesSpec +Container of table level partitioned checks, divided by the time window (daily, monthly, etc.) + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`daily`](./tables.md#tabledailypartitionedcheckcategoriesspec)|Configuration of day partitioned data quality checks evaluated at a table level.|*[TableDailyPartitionedCheckCategoriesSpec](./tables.md#tabledailypartitionedcheckcategoriesspec)*| +|[`monthly`](./tables.md#tablemonthlypartitionedcheckcategoriesspec)|Configuration of monthly partitioned data quality checks evaluated at a table level..|*[TableMonthlyPartitionedCheckCategoriesSpec](./tables.md#tablemonthlypartitionedcheckcategoriesspec)*| + + +___ + +## TableQualityPolicySpec +The default configuration of table-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on tables. + This configuration serves as a data quality policy that defines the data quality checks that are verified on matching tables. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`priority`|The priority of the pattern. Patterns with lower values are applied before patterns with higher priority values.|*integer*| +|`disabled`|Disables this data quality check configuration. The checks will not be activated.|*boolean*| +|`description`|The description (documentation) of this data quality check configuration.|*string*| +|[`target`](./table_quality_policies.md#targettablepatternspec)|The target table filter that are filtering the table and connection on which the default checks are applied.|*[TargetTablePatternSpec](./table_quality_policies.md#targettablepatternspec)*| +|[`profiling_checks`](./tables.md#tableprofilingcheckcategoriesspec)|Configuration of data quality profiling checks that are enabled. Pick a check from a category, apply the parameters and rules to enable it.|*[TableProfilingCheckCategoriesSpec](./tables.md#tableprofilingcheckcategoriesspec)*| +|[`monitoring_checks`](#tablemonitoringcheckcategoriesspec)|Configuration of table level monitoring checks. Monitoring checks are data quality checks that are evaluated for each period of time (daily, weekly, monthly, etc.). A monitoring check stores only the most recent data quality check result for each period of time.|*[TableMonitoringCheckCategoriesSpec](#tablemonitoringcheckcategoriesspec)*| +|[`partitioned_checks`](#tablepartitionedcheckcategoriesspec)|Configuration of table level date/time partitioned checks. Partitioned data quality checks are evaluated for each partition separately, raising separate alerts at a partition level. The table does not need to be physically partitioned by date, it is possible to run data quality checks for each day or month of data separately.|*[TablePartitionedCheckCategoriesSpec](#tablepartitionedcheckcategoriesspec)*| + + +___ + +## TableQualityPolicyModel +Default table-level checks pattern model that is returned by the REST API. Describes a configuration of data quality checks for a named pattern. DQOps applies these checks on tables that match the filter. + + +**The structure of this object is described below** + + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`policy_name`|Quality policy name|*string*| +|[`policy_spec`](#tablequalitypolicyspec)|The quality policy specification.|*[TableQualityPolicySpec](#tablequalitypolicyspec)*| +|`can_edit`|Boolean flag that decides if the current user can update or delete this object.|*boolean*| +|`yaml_parsing_error`|Optional parsing error that was captured when parsing the YAML file. This field is null when the YAML file is valid. If an error was captured, this field returns the file parsing error message and the file location.|*string*| + + +___ + diff --git a/docs/client/models/tables.md b/docs/client/models/tables.md index 5f0b442e2e..821d3ded0f 100644 --- a/docs/client/models/tables.md +++ b/docs/client/models/tables.md @@ -50,6 +50,7 @@ Container of table level daily monitoring. Contains categories of daily monitori |[`custom_sql`](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tablecustomsqldailymonitoringchecksspec)|Daily monitoring custom SQL checks|*[TableCustomSqlDailyMonitoringChecksSpec](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tablecustomsqldailymonitoringchecksspec)*| |[`availability`](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableavailabilitydailymonitoringchecksspec)|Daily monitoring table availability checks|*[TableAvailabilityDailyMonitoringChecksSpec](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableavailabilitydailymonitoringchecksspec)*| |[`schema`](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableschemadailymonitoringchecksspec)|Daily monitoring table schema checks|*[TableSchemaDailyMonitoringChecksSpec](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableschemadailymonitoringchecksspec)*| +|[`uniqueness`](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableuniquenessdailymonitoringchecksspec)|Daily monitoring uniqueness checks on a table level.|*[TableUniquenessDailyMonitoringChecksSpec](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableuniquenessdailymonitoringchecksspec)*| |[`comparisons`](#tablecomparisondailymonitoringchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonDailyMonitoringChecksSpecMap](#tablecomparisondailymonitoringchecksspecmap)*| |[`custom`](#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](#customcheckspecmap)*| @@ -84,6 +85,7 @@ Container of table level daily partitioned checks. Contains categories of daily |[`volume`](../../reference/yaml/partitioned/table-daily-partitioned-checks.md#tablevolumedailypartitionedchecksspec)|Volume daily partitioned data quality checks that verify the quality of every day of data separately|*[TableVolumeDailyPartitionedChecksSpec](../../reference/yaml/partitioned/table-daily-partitioned-checks.md#tablevolumedailypartitionedchecksspec)*| |[`timeliness`](../../reference/yaml/partitioned/table-daily-partitioned-checks.md#tabletimelinessdailypartitionedchecksspec)|Daily partitioned timeliness checks|*[TableTimelinessDailyPartitionedChecksSpec](../../reference/yaml/partitioned/table-daily-partitioned-checks.md#tabletimelinessdailypartitionedchecksspec)*| |[`custom_sql`](../../reference/yaml/partitioned/table-daily-partitioned-checks.md#tablecustomsqldailypartitionedchecksspec)|Custom SQL daily partitioned data quality checks that verify the quality of every day of data separately|*[TableCustomSqlDailyPartitionedChecksSpec](../../reference/yaml/partitioned/table-daily-partitioned-checks.md#tablecustomsqldailypartitionedchecksspec)*| +|[`uniqueness`](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableuniquenessdailypartitionchecksspec)|Daily partitioned uniqueness checks on a table level.|*[TableUniquenessDailyPartitionChecksSpec](../../reference/yaml/monitoring/table-daily-monitoring-checks.md#tableuniquenessdailypartitionchecksspec)*| |[`comparisons`](#tablecomparisondailypartitionedchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonDailyPartitionedChecksSpecMap](#tablecomparisondailypartitionedchecksspecmap)*| |[`custom`](#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](#customcheckspecmap)*| @@ -139,6 +141,7 @@ Container of table level monthly monitoring checks. Contains categories of month |[`custom_sql`](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tablecustomsqlmonthlymonitoringchecksspec)|Monthly monitoring of custom SQL checks|*[TableCustomSqlMonthlyMonitoringChecksSpec](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tablecustomsqlmonthlymonitoringchecksspec)*| |[`availability`](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableavailabilitymonthlymonitoringchecksspec)|Daily partitioned availability checks|*[TableAvailabilityMonthlyMonitoringChecksSpec](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableavailabilitymonthlymonitoringchecksspec)*| |[`schema`](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableschemamonthlymonitoringchecksspec)|Monthly monitoring table schema checks|*[TableSchemaMonthlyMonitoringChecksSpec](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableschemamonthlymonitoringchecksspec)*| +|[`uniqueness`](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableuniquenessmonthlymonitoringchecksspec)|Monthly monitoring uniqueness checks on a table level.|*[TableUniquenessMonthlyMonitoringChecksSpec](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableuniquenessmonthlymonitoringchecksspec)*| |[`comparisons`](#tablecomparisonmonthlymonitoringchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonMonthlyMonitoringChecksSpecMap](#tablecomparisonmonthlymonitoringchecksspecmap)*| |[`custom`](#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](#customcheckspecmap)*| @@ -173,6 +176,7 @@ Container of table level monthly partitioned checks. Contains categories of mont |[`volume`](../../reference/yaml/partitioned/table-monthly-partitioned-checks.md#tablevolumemonthlypartitionedchecksspec)|Volume monthly partitioned data quality checks that verify the quality of every month of data separately|*[TableVolumeMonthlyPartitionedChecksSpec](../../reference/yaml/partitioned/table-monthly-partitioned-checks.md#tablevolumemonthlypartitionedchecksspec)*| |[`timeliness`](../../reference/yaml/partitioned/table-monthly-partitioned-checks.md#tabletimelinessmonthlypartitionedchecksspec)|Monthly partitioned timeliness checks|*[TableTimelinessMonthlyPartitionedChecksSpec](../../reference/yaml/partitioned/table-monthly-partitioned-checks.md#tabletimelinessmonthlypartitionedchecksspec)*| |[`custom_sql`](../../reference/yaml/partitioned/table-monthly-partitioned-checks.md#tablecustomsqlmonthlypartitionedchecksspec)|Custom SQL monthly partitioned data quality checks that verify the quality of every month of data separately|*[TableCustomSqlMonthlyPartitionedChecksSpec](../../reference/yaml/partitioned/table-monthly-partitioned-checks.md#tablecustomsqlmonthlypartitionedchecksspec)*| +|[`uniqueness`](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableuniquenessmonthlypartitionchecksspec)|Monthly partitioned uniqueness checks on a table level.|*[TableUniquenessMonthlyPartitionChecksSpec](../../reference/yaml/monitoring/table-monthly-monitoring-checks.md#tableuniquenessmonthlypartitionchecksspec)*| |[`comparisons`](#tablecomparisonmonthlypartitionedchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonMonthlyPartitionedChecksSpecMap](#tablecomparisonmonthlypartitionedchecksspecmap)*| |[`custom`](#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](#customcheckspecmap)*| @@ -228,6 +232,7 @@ Container of table level checks that are activated on a table level. |[`custom_sql`](../../reference/yaml/profiling/table-profiling-checks.md#tablecustomsqlprofilingchecksspec)|Configuration of data quality checks that are evaluating custom SQL conditions and aggregated expressions.|*[TableCustomSqlProfilingChecksSpec](../../reference/yaml/profiling/table-profiling-checks.md#tablecustomsqlprofilingchecksspec)*| |[`availability`](../../reference/yaml/profiling/table-profiling-checks.md#tableavailabilityprofilingchecksspec)|Configuration of the table availability data quality checks on a table level.|*[TableAvailabilityProfilingChecksSpec](../../reference/yaml/profiling/table-profiling-checks.md#tableavailabilityprofilingchecksspec)*| |[`schema`](../../reference/yaml/profiling/table-profiling-checks.md#tableschemaprofilingchecksspec)|Configuration of schema (column count and schema) data quality checks on a table level.|*[TableSchemaProfilingChecksSpec](../../reference/yaml/profiling/table-profiling-checks.md#tableschemaprofilingchecksspec)*| +|[`uniqueness`](../../reference/yaml/profiling/table-profiling-checks.md#tableuniquenessprofilingchecksspec)|Configuration of uniqueness checks on a table level.|*[TableUniquenessProfilingChecksSpec](../../reference/yaml/profiling/table-profiling-checks.md#tableuniquenessprofilingchecksspec)*| |[`comparisons`](#tablecomparisonprofilingchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonProfilingChecksSpecMap](#tablecomparisonprofilingchecksspecmap)*| |[`custom`](#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](#customcheckspecmap)*| diff --git a/docs/client/models/users.md b/docs/client/models/users.md index d5075c9f12..755a6b0fc0 100644 --- a/docs/client/models/users.md +++ b/docs/client/models/users.md @@ -5,8 +5,8 @@ title: DQOps REST API users models reference The references of all objects used by [users](../operations/users.md) REST API operations are listed below. -## DqoCloudUserModel -DQOps Cloud user model - identifies a user in a multi-user DQOps deployment. +## DqoUserRolesModel +DQOps user model - identifies a user in a multi-user DQOps deployment and the user's roles. **The structure of this object is described below** @@ -15,7 +15,8 @@ DQOps Cloud user model - identifies a user in a multi-user DQOps deployment. | Property name | Description                     | Data type | |---------------|---------------------------------|-----------| |`email`|User's email that identifies the user.|*string*| -|[`account_role`](./environment.md#dqouserrole)|Account role.|*[DqoUserRole](./environment.md#dqouserrole)*| +|[`account_role`](./environment.md#dqouserrole)|User role at the whole account level. This role is applicable to all data domains.|*[DqoUserRole](./environment.md#dqouserrole)*| +|`data_domain_roles`|User roles within each data domain. Data domains are supported in an ENTERPRISE version of DQOps and they are managed by the SaaS components of DQOps Cloud.|*Dict[string, [DqoUserRole](./environment.md#dqouserrole)]*| ___ diff --git a/docs/client/operations/check_results.md b/docs/client/operations/check_results.md index cb83c18238..190b87527a 100644 --- a/docs/client/operations/check_results.md +++ b/docs/client/operations/check_results.md @@ -6303,7 +6303,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "dimensions" : { } } }, - "dimensions" : { } + "dimensions" : { }, + "table_exist" : true } ``` @@ -6500,7 +6501,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, dimensions={ - } + }, + table_exist=True ) ``` @@ -6698,7 +6700,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, dimensions={ - } + }, + table_exist=True ) ``` @@ -6899,7 +6902,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, dimensions={ - } + }, + table_exist=True ) ``` @@ -7100,7 +7104,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, dimensions={ - } + }, + table_exist=True ) ``` diff --git a/docs/client/operations/column_quality_policies.md b/docs/client/operations/column_quality_policies.md new file mode 100644 index 0000000000..24e519b8b1 --- /dev/null +++ b/docs/client/operations/column_quality_policies.md @@ -0,0 +1,5373 @@ +--- +title: DQOps REST API column_quality_policies operations +--- +# DQOps REST API column_quality_policies operations +Operations for managing the configuration of data quality policies at a column level. Policies are the default configuration of data quality checks for columns matching a pattern. + + +___ +## copy_from_column_quality_policy +Creates (adds) a copy of an existing default column-level checks pattern configuration, under a new name. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/copy_from_column_quality_policy.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/policies/checks/column/{targetPatternName}/copyfrom/{sourcePatternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`target_pattern_name`|Target pattern name|*string*|:material-check-bold:| +|`source_pattern_name`|Source pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/policies/checks/column/default/copyfrom/default^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import copy_from_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = copy_from_column_quality_policy.sync( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import copy_from_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = await copy_from_column_quality_policy.asyncio( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import copy_from_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = copy_from_column_quality_policy.sync( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import copy_from_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = await copy_from_column_quality_policy.asyncio( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + + +___ +## create_column_quality_policy +Creates (adds) a new default column-level checks pattern configuration by saving a full specification object. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/policies/checks/column/{patternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model|*[ColumnQualityPolicyModel](../models/column_quality_policies.md#columnqualitypolicymodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/policies/checks/column/default^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"id columns not null\",\"policy_spec\":{\"priority\":1000,\"target\":{\"column\":\"id\"},\"monitoring_checks\":{\"daily\":{\"nulls\":{\"daily_nulls_count\":{\"error\":{\"max_count\":0}}}}}},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = create_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await create_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = create_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await create_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## create_column_quality_policy_target +Creates (adds) a new default column-level checks pattern configuration. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy_target.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/target +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model with only the target filters|*[ColumnQualityPolicyListModel](../models/column_quality_policies.md#columnqualitypolicylistmodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/policies/checks/column/default/target^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"default\",\"priority\":100,\"disabled\":false,\"target_column\":{\"connection\":\"dwh\",\"column\":\"id\"},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = create_column_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = await create_column_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = create_column_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import create_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = await create_column_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## delete_column_quality_policy +Deletes a default column-level checks pattern + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/delete_column_quality_policy.py) to see the source code on GitHub. + + +**DELETE** +``` +http://localhost:8888/api/policies/checks/column/{patternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X DELETE http://localhost:8888/api/policies/checks/column/default^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import delete_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = delete_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import delete_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = await delete_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import delete_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = delete_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import delete_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = await delete_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + + + +___ +## get_column_quality_policies +Returns a flat list of all column-level default check patterns configured for this instance. Default checks are applied on columns dynamically. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policies.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`column_quality_policy_list_model`||*List[[ColumnQualityPolicyListModel](../models/column_quality_policies.md#columnqualitypolicylistmodel)]*| + + + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + [ { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_column" : { + "connection" : "dwh", + "column" : "id" + }, + "can_edit" : true + }, { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_column" : { + "connection" : "dwh", + "column" : "id" + }, + "can_edit" : true + }, { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_column" : { + "connection" : "dwh", + "column" : "id" + }, + "can_edit" : true + } ] + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policies + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_column_quality_policies.sync( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ] + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policies + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_column_quality_policies.asyncio( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ] + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policies + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_column_quality_policies.sync( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ] + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policies + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_column_quality_policies.asyncio( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ), + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ] + ``` + + + + + + +___ +## get_column_quality_policy +Returns a default checks pattern definition as a full specification object + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column/{patternName} +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`column_quality_policy_model`](../models/column_quality_policies.md#columnqualitypolicymodel)||*[ColumnQualityPolicyModel](../models/column_quality_policies.md#columnqualitypolicymodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Column pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column/default^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "policy_name" : "id columns not null", + "policy_spec" : { + "priority" : 1000, + "target" : { + "column" : "id" + }, + "monitoring_checks" : { + "daily" : { + "nulls" : { + "daily_nulls_count" : { + "error" : { + "max_count" : 0 + } + } + } + } + } + }, + "can_edit" : true + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + + +___ +## get_column_quality_policy_target +Returns a default checks pattern definition + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policy_target.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/target +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`column_quality_policy_list_model`](../models/column_quality_policies.md#columnqualitypolicylistmodel)||*[ColumnQualityPolicyListModel](../models/column_quality_policies.md#columnqualitypolicylistmodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Column pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column/default/target^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_column" : { + "connection" : "dwh", + "column" : "id" + }, + "can_edit" : true + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy_target + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_column_quality_policy_target.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy_target + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_column_quality_policy_target.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy_target + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_column_quality_policy_target.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_column_quality_policy_target + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_column_quality_policy_target.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + ``` + + + + + + +___ +## get_monitoring_daily_column_quality_policy +Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a column level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_monitoring_daily_column_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/monitoring/daily +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column/default/monitoring/daily^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_daily_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_daily_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_daily_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_daily_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_daily_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_daily_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_monitoring_monthly_column_quality_policy +Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a column level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_monitoring_monthly_column_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/monitoring/monthly +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column/default/monitoring/monthly^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_monthly_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_monthly_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_monthly_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_monthly_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_monthly_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_monitoring_monthly_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_partitioned_daily_column_quality_policy +Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a column level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_partitioned_daily_column_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/partitioned/daily +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column/default/partitioned/daily^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_daily_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_daily_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_daily_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_daily_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_daily_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_daily_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_partitioned_monthly_column_quality_policy +Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a column level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_partitioned_monthly_column_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/partitioned/monthly +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column/default/partitioned/monthly^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_monthly_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_monthly_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_monthly_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_monthly_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_monthly_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_partitioned_monthly_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_profiling_column_quality_policy +Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a column level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/get_profiling_column_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/profiling +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/column/default/profiling^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_profiling_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_profiling_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_profiling_column_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_profiling_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_profiling_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_profiling_column_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import get_profiling_column_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_profiling_column_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## update_column_quality_policy +Updates an default column-level checks pattern by saving a full specification object + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/column/{patternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model|*[ColumnQualityPolicyModel](../models/column_quality_policies.md#columnqualitypolicymodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/column/default^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"id columns not null\",\"policy_spec\":{\"priority\":1000,\"target\":{\"column\":\"id\"},\"monitoring_checks\":{\"daily\":{\"nulls\":{\"daily_nulls_count\":{\"error\":{\"max_count\":0}}}}}},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = update_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await update_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = update_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy + from dqops.client.models import ColumnComparisonProfilingChecksSpecMap, \ + ColumnMonitoringCheckCategoriesSpec, \ + ColumnNullsCountCheckSpec, \ + ColumnNullsNullsCountSensorParametersSpec, \ + ColumnNullsProfilingChecksSpec, \ + ColumnPartitionedCheckCategoriesSpec, \ + ColumnProfilingCheckCategoriesSpec, \ + ColumnQualityPolicyModel, \ + ColumnQualityPolicySpec, \ + MaxCountRule0ErrorParametersSpec, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyModel( + policy_name='id columns not null', + policy_spec=ColumnQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetColumnPatternSpec(column='id'), + profiling_checks=ColumnProfilingCheckCategoriesSpec(comparisons=ColumnComparisonProfilingChecksSpecMap()), + monitoring_checks=ColumnMonitoringCheckCategoriesSpec( + daily=ColumnDailyMonitoringCheckCategoriesSpec( + nulls=ColumnNullsDailyMonitoringChecksSpec( + daily_nulls_count=ColumnNullsCountCheckSpec( + parameters=ColumnNullsNullsCountSensorParametersSpec(), + error=MaxCountRule0ErrorParametersSpec(max_count=0), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=ColumnComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=ColumnPartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await update_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_column_quality_policy_target +Updates an default column-level checks pattern, changing only the target object + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/update_column_quality_policy_target.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/target +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model|*[ColumnQualityPolicyListModel](../models/column_quality_policies.md#columnqualitypolicylistmodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/column/default/target^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"default\",\"priority\":100,\"disabled\":false,\"target_column\":{\"connection\":\"dwh\",\"column\":\"id\"},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = update_column_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = await update_column_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = update_column_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_column_quality_policy_target + from dqops.client.models import ColumnQualityPolicyListModel, \ + TargetColumnPatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = ColumnQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_column=TargetColumnPatternSpec( + column='id', + connection='dwh' + ), + can_edit=True + ) + + call_result = await update_column_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_monitoring_daily_column_quality_policy +New configuration of the default daily monitoring checks on a column level. These checks will be applied to columns. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/update_monitoring_daily_column_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/monitoring/daily +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality daily monitoring checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/column/default/monitoring/daily^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_daily_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_daily_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_monitoring_monthly_column_quality_policy +New configuration of the default monthly monitoring checks on a column level. These checks will be applied to columns. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/update_monitoring_monthly_column_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/monitoring/monthly +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality monthly monitoring checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/column/default/monitoring/monthly^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_monthly_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_monthly_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_monitoring_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_partitioned_daily_column_quality_policy +New configuration of the default daily partitioned checks on a column level. These checks will be applied to columns. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/update_partitioned_daily_column_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/partitioned/daily +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality daily partitioned checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/column/default/partitioned/daily^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_daily_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_daily_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_daily_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_daily_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_partitioned_monthly_column_quality_policy +New configuration of the default monthly partitioned checks on a column level. These checks will be applied to columns. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/update_partitioned_monthly_column_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/partitioned/monthly +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality monthly partitioned checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/column/default/partitioned/monthly^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_monthly_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_monthly_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_partitioned_monthly_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_monthly_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_profiling_column_quality_policy +New configuration of the default profiling checks on a column level. These checks will be applied to columns. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/column_quality_policies/update_profiling_column_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/column/{patternName}/profiling +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality profiling checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/column/default/profiling^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_profiling_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_profiling_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_profiling_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_profiling_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_profiling_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_profiling_column_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.column_quality_policies import update_profiling_column_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_profiling_column_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + diff --git a/docs/client/operations/columns.md b/docs/client/operations/columns.md index 14de265152..46f9af8c3d 100644 --- a/docs/client/operations/columns.md +++ b/docs/client/operations/columns.md @@ -97,7 +97,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = create_column.sync( @@ -153,7 +156,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = await create_column.asyncio( @@ -212,7 +218,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = create_column.sync( @@ -271,7 +280,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = await create_column.asyncio( @@ -632,7 +644,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ), can_edit=True ) @@ -695,7 +710,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ), can_edit=True ) @@ -761,7 +779,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ), can_edit=True ) @@ -827,7 +848,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ), can_edit=True ) @@ -904,6 +928,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "nullable" : false, "length" : 256 }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -958,6 +983,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1013,6 +1041,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1071,6 +1102,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1129,6 +1163,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -6929,6 +6966,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "nullable" : false, "length" : 256 }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -6947,6 +6985,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "nullable" : false, "length" : 256 }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -6965,6 +7004,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "nullable" : false, "length" : 256 }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -7020,6 +7060,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7043,6 +7086,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7066,6 +7112,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7123,6 +7172,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7146,6 +7198,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7169,6 +7224,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7229,6 +7287,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7252,6 +7313,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7275,6 +7339,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7335,6 +7402,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7358,6 +7428,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -7381,6 +7454,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -8455,7 +8531,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = update_column.sync( @@ -8511,7 +8590,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = await update_column.asyncio( @@ -8570,7 +8652,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = update_column.sync( @@ -8629,7 +8714,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ) ), comparisons=ColumnComparisonProfilingChecksSpecMap() - ) + ), + advanced_properties={ + + } ) call_result = await update_column.asyncio( @@ -8693,7 +8781,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"connection_name\":\"sample_connection\",\"table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"column_name\":\"sample_column\",\"has_any_configured_checks\":true,\"has_any_configured_profiling_checks\":true,\"type_snapshot\":{\"column_type\":\"string\",\"nullable\":false,\"length\":256},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" + "{\"connection_name\":\"sample_connection\",\"table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"column_name\":\"sample_column\",\"has_any_configured_checks\":true,\"has_any_configured_profiling_checks\":true,\"type_snapshot\":{\"column_type\":\"string\",\"nullable\":false,\"length\":256},\"advanced_properties\":{},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" ``` @@ -8732,6 +8820,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -8784,6 +8875,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -8839,6 +8933,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -8894,6 +8991,9 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, diff --git a/docs/client/operations/connections.md b/docs/client/operations/connections.md index 05240ab736..5dea4f6f28 100644 --- a/docs/client/operations/connections.md +++ b/docs/client/operations/connections.md @@ -592,7 +592,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = create_connection.sync( @@ -643,7 +646,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = await create_connection.asyncio( @@ -697,7 +703,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = create_connection.sync( @@ -751,7 +760,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = await create_connection.asyncio( @@ -809,7 +821,7 @@ http://localhost:8888/api/connections/{connectionName}/basic -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"connection_name\":\"sample_connection\",\"parallel_jobs_limit\":4,\"provider_type\":\"postgresql\",\"postgresql\":{\"host\":\"localhost\",\"port\":\"5432\",\"database\":\"db\",\"user\":\"PASSWD\",\"sslmode\":\"disable\"},\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"partitioned\"},\"collect_statistics_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"columnNames\":[]},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":false,\"deleteIncidents\":false},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" + "{\"connection_name\":\"sample_connection\",\"parallel_jobs_limit\":4,\"provider_type\":\"postgresql\",\"postgresql\":{\"host\":\"localhost\",\"port\":\"5432\",\"database\":\"db\",\"user\":\"PASSWD\",\"sslmode\":\"disable\"},\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"partitioned\"},\"collect_statistics_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"columnNames\":[]},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"deleteChecksConfiguration\":false},\"advanced_properties\":{},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" ``` @@ -877,9 +889,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -958,9 +974,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1042,9 +1062,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1126,9 +1150,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1423,9 +1451,11 @@ http://localhost:8888/api/connections "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -1471,9 +1501,11 @@ http://localhost:8888/api/connections "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -1519,9 +1551,11 @@ http://localhost:8888/api/connections "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -1598,9 +1632,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1649,9 +1687,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1700,9 +1742,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1781,9 +1827,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1832,9 +1882,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1883,9 +1937,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -1967,9 +2025,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2018,9 +2080,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2069,9 +2135,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2153,9 +2223,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2204,9 +2278,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2255,9 +2333,13 @@ http://localhost:8888/api/connections delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2545,9 +2627,11 @@ http://localhost:8888/api/connections/{connectionName}/basic "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -2624,9 +2708,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2704,9 +2792,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2787,9 +2879,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -2870,9 +2966,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -4294,7 +4394,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = update_connection.sync( @@ -4345,7 +4448,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = await update_connection.asyncio( @@ -4399,7 +4505,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = update_connection.sync( @@ -4453,7 +4562,10 @@ http://localhost:8888/api/connections/{connectionName} max_incident_length_days=60, mute_for_days=60, disabled=False - ) + ), + advanced_properties={ + + } ) call_result = await update_connection.asyncio( @@ -4511,7 +4623,7 @@ http://localhost:8888/api/connections/{connectionName}/basic -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"connection_name\":\"sample_connection\",\"parallel_jobs_limit\":4,\"provider_type\":\"postgresql\",\"postgresql\":{\"host\":\"localhost\",\"port\":\"5432\",\"database\":\"db\",\"user\":\"PASSWD\",\"sslmode\":\"disable\"},\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"partitioned\"},\"collect_statistics_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"columnNames\":[]},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":false,\"deleteIncidents\":false},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" + "{\"connection_name\":\"sample_connection\",\"parallel_jobs_limit\":4,\"provider_type\":\"postgresql\",\"postgresql\":{\"host\":\"localhost\",\"port\":\"5432\",\"database\":\"db\",\"user\":\"PASSWD\",\"sslmode\":\"disable\"},\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"partitioned\"},\"collect_statistics_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"columnNames\":[]},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"deleteChecksConfiguration\":false},\"advanced_properties\":{},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" ``` @@ -4579,9 +4691,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -4660,9 +4776,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -4744,9 +4864,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -4828,9 +4952,13 @@ http://localhost:8888/api/connections/{connectionName}/basic delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, diff --git a/docs/client/operations/data_domains.md b/docs/client/operations/data_domains.md new file mode 100644 index 0000000000..5d65894ecc --- /dev/null +++ b/docs/client/operations/data_domains.md @@ -0,0 +1,949 @@ +--- +title: DQOps REST API data_domains operations +--- +# DQOps REST API data_domains operations +Data domain management API to create different data domains. + + +___ +## create_data_domain +Creates a new data domain given a data domain display name. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_domains/create_data_domain.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/domains/ +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`local_data_domain_model`](../models/data_domains.md#localdatadomainmodel)||*[LocalDataDomainModel](../models/data_domains.md#localdatadomainmodel)*| + + + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Data domain display name|*string*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/domains/^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "\"sample_string_value\"" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "domain_name" : "sales", + "display_name" : "Sales data domain", + "created_at" : "2024-09-14T18:33:00Z", + "enable_scheduler" : true + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import create_data_domain + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = 'sample_string_value' + + call_result = create_data_domain.sync( + client=dqops_client, + json_body=request_body + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import create_data_domain + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = 'sample_string_value' + + call_result = await create_data_domain.asyncio( + client=dqops_client, + json_body=request_body + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import create_data_domain + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = 'sample_string_value' + + call_result = create_data_domain.sync( + client=dqops_client, + json_body=request_body + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import create_data_domain + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = 'sample_string_value' + + call_result = await create_data_domain.asyncio( + client=dqops_client, + json_body=request_body + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ``` + + + + + + +___ +## delete_data_domain +Deletes a data domain. The domain is deleted in the DQOps SaaS cloud and locally. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_domains/delete_data_domain.py) to see the source code on GitHub. + + +**DELETE** +``` +http://localhost:8888/api/domains/{dataDomainName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`data_domain_name`|Data domain name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X DELETE http://localhost:8888/api/domains/crm^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import delete_data_domain + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = delete_data_domain.sync( + 'crm', + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import delete_data_domain + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = await delete_data_domain.asyncio( + 'crm', + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import delete_data_domain + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = delete_data_domain.sync( + 'crm', + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import delete_data_domain + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = await delete_data_domain.asyncio( + 'crm', + client=dqops_client + ) + + ``` + + + + + +___ +## get_local_data_domains +Returns a list of local data domains that this instance is maintaining. Data domains are supported only in an ENTERPRISE versions of DQOps. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_domains/get_local_data_domains.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/domains/ +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`local_data_domain_model`||*List[[LocalDataDomainModel](../models/data_domains.md#localdatadomainmodel)]*| + + + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/domains/^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + [ { + "domain_name" : "sales", + "display_name" : "Sales data domain", + "created_at" : "2024-09-14T18:33:00Z", + "enable_scheduler" : true + }, { + "domain_name" : "sales", + "display_name" : "Sales data domain", + "created_at" : "2024-09-14T18:33:00Z", + "enable_scheduler" : true + }, { + "domain_name" : "sales", + "display_name" : "Sales data domain", + "created_at" : "2024-09-14T18:33:00Z", + "enable_scheduler" : true + } ] + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import get_local_data_domains + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_local_data_domains.sync( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ] + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import get_local_data_domains + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_local_data_domains.asyncio( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ] + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import get_local_data_domains + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_local_data_domains.sync( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ] + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import get_local_data_domains + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_local_data_domains.asyncio( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ), + LocalDataDomainModel( + domain_name='sales', + display_name='Sales data domain', + created_at=OffsetDateTime( + offset=ZoneOffset( + total_seconds=0, + id='Z' + ) + ), + enable_scheduler=True + ) + ] + ``` + + + + + + +___ +## switch_to_data_domain +Switches to a different data domain. This operation sends a special cookie and redirects the user to the home screen. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_domains/switch_to_data_domain.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/domains/{dataDomainName}/switch +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`data_domain_name`|Data domain name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/domains/crm/switch^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import switch_to_data_domain + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = switch_to_data_domain.sync( + 'crm', + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import switch_to_data_domain + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await switch_to_data_domain.asyncio( + 'crm', + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import switch_to_data_domain + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = switch_to_data_domain.sync( + 'crm', + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import switch_to_data_domain + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await switch_to_data_domain.asyncio( + 'crm', + client=dqops_client + ) + + ``` + + + + + +___ +## synchronize_data_domains +Synchronizes the domains in the SaaS cloud to this instance. All data domains will be created locally. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_domains/synchronize_data_domains.py) to see the source code on GitHub. + + +**PATCH** +``` +http://localhost:8888/api/domains/ +``` + + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PATCH http://localhost:8888/api/domains/^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import synchronize_data_domains + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = synchronize_data_domains.sync( + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import synchronize_data_domains + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = await synchronize_data_domains.asyncio( + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import synchronize_data_domains + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = synchronize_data_domains.sync( + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_domains import synchronize_data_domains + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = await synchronize_data_domains.asyncio( + client=dqops_client + ) + + ``` + + + + + diff --git a/docs/client/operations/data_lineage.md b/docs/client/operations/data_lineage.md new file mode 100644 index 0000000000..dc4b61c451 --- /dev/null +++ b/docs/client/operations/data_lineage.md @@ -0,0 +1,1139 @@ +--- +title: DQOps REST API data_lineage operations +--- +# DQOps REST API data_lineage operations +Operations related to managing and inspecting table and column lineage. + + +___ +## create_table_source_table +Creates a new source table of the table's data lineage. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_lineage/create_table_source_table.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`connection_name`|Connection name|*string*|:material-check-bold:| +|`schema_name`|Schema name|*string*|:material-check-bold:| +|`table_name`|Table name|*string*|:material-check-bold:| +|`source_connection`|Source connection name|*string*|:material-check-bold:| +|`source_schema`|Source schema name|*string*|:material-check-bold:| +|`source_table`|Source table name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Table lineage source list model|*[TableLineageSourceSpec](../models/data_lineage.md#tablelineagesourcespec)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/connections/sample_connection/schemas/sample_schema/tables/sample_table/lineage/from/sample_connection/schemas/sample_schema/tables/sample_table^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"source_connection\":\"\",\"source_schema\":\"\",\"source_table\":\"\"}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import create_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = create_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import create_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = await create_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import create_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = create_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import create_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = await create_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## delete_table_source_table +Deletes a specific data lineage source table of the given table. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_lineage/delete_table_source_table.py) to see the source code on GitHub. + + +**DELETE** +``` +http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`connection_name`|Connection name|*string*|:material-check-bold:| +|`schema_name`|Schema name|*string*|:material-check-bold:| +|`table_name`|Table name|*string*|:material-check-bold:| +|`source_connection`|Source connection name|*string*|:material-check-bold:| +|`source_schema`|Source schema name|*string*|:material-check-bold:| +|`source_table`|Source table name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X DELETE http://localhost:8888/api/connections/sample_connection/schemas/sample_schema/tables/sample_table/lineage/from/sample_connection/schemas/sample_schema/tables/sample_table^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import delete_table_source_table + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = delete_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import delete_table_source_table + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = await delete_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import delete_table_source_table + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = delete_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import delete_table_source_table + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = await delete_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + + + +___ +## get_table_source_table +Reads a specific data lineage source table defined on a target tale. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_lineage/get_table_source_table.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable} +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`table_lineage_source_spec`](../models/data_lineage.md#tablelineagesourcespec)||*[TableLineageSourceSpec](../models/data_lineage.md#tablelineagesourcespec)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`connection_name`|Connection name|*string*|:material-check-bold:| +|`schema_name`|Schema name|*string*|:material-check-bold:| +|`table_name`|Table name|*string*|:material-check-bold:| +|`source_connection`|Source connection name|*string*|:material-check-bold:| +|`source_schema`|Source schema name|*string*|:material-check-bold:| +|`source_table`|Source table name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/connections/sample_connection/schemas/sample_schema/tables/sample_table/lineage/from/sample_connection/schemas/sample_schema/tables/sample_table^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "source_connection" : "", + "source_schema" : "", + "source_table" : "" + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_table + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_table + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_table + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_table + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + ``` + + + + + + +___ +## get_table_source_tables +Returns a list of source tables on the data lineage that are sources of the given table. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_lineage/get_table_source_tables.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`table_lineage_source_list_model`||*List[[TableLineageSourceListModel](../models/data_lineage.md#tablelineagesourcelistmodel)]*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`connection_name`|Connection name|*string*|:material-check-bold:| +|`schema_name`|Schema name|*string*|:material-check-bold:| +|`table_name`|Table name|*string*|:material-check-bold:| +|[`check_type`](../models/common.md#checktype)|Optional parameter for the check type, when provided, returns the results for data quality dimensions for the data quality checks of that type|*[CheckType](../models/common.md#checktype)*| | + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/connections/sample_connection/schemas/sample_schema/tables/sample_table/lineage^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + [ { + "target_connection" : "datalake", + "target_schema" : "landing_app", + "target_table" : "customers_landing", + "source_connection" : "sourcedb", + "source_schema" : "app", + "source_table" : "t_customers", + "can_edit" : false + }, { + "target_connection" : "datalake", + "target_schema" : "landing_app", + "target_table" : "customers_landing", + "source_connection" : "sourcedb", + "source_schema" : "app", + "source_table" : "t_customers", + "can_edit" : false + }, { + "target_connection" : "datalake", + "target_schema" : "landing_app", + "target_table" : "customers_landing", + "source_connection" : "sourcedb", + "source_schema" : "app", + "source_table" : "t_customers", + "can_edit" : false + } ] + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_tables + from dqops.client.models import CheckType + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_table_source_tables.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ) + ] + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_tables + from dqops.client.models import CheckType + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_table_source_tables.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ) + ] + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_tables + from dqops.client.models import CheckType + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_table_source_tables.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ) + ] + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import get_table_source_tables + from dqops.client.models import CheckType + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_table_source_tables.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ), + TableLineageSourceListModel( + target_connection='datalake', + target_schema='landing_app', + target_table='customers_landing', + source_connection='sourcedb', + source_schema='app', + source_table='t_customers', + can_edit=False + ) + ] + ``` + + + + + + +___ +## update_table_source_table +Update a specific data lineage source table using a new model. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/data_lineage/update_table_source_table.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tables/{tableName}/lineage/from/{sourceConnection}/schemas/{sourceSchema}/tables/{sourceTable} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`connection_name`|Connection name|*string*|:material-check-bold:| +|`schema_name`|Schema name|*string*|:material-check-bold:| +|`table_name`|Table name|*string*|:material-check-bold:| +|`source_connection`|Source connection name|*string*|:material-check-bold:| +|`source_schema`|Source schema name|*string*|:material-check-bold:| +|`source_table`|Source table name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Table lineage source list model|*[TableLineageSourceSpec](../models/data_lineage.md#tablelineagesourcespec)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/connections/sample_connection/schemas/sample_schema/tables/sample_table/lineage/from/sample_connection/schemas/sample_schema/tables/sample_table^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"source_connection\":\"\",\"source_schema\":\"\",\"source_table\":\"\"}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import update_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = update_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import update_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = await update_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import update_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = update_table_source_table.sync( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.data_lineage import update_table_source_table + from dqops.client.models import ColumnLineageSourceSpecMap, \ + TableLineageSourceSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableLineageSourceSpec( + source_connection='', + source_schema='', + source_table='', + columns=ColumnLineageSourceSpecMap() + ) + + call_result = await update_table_source_table.asyncio( + 'sample_connection', + 'sample_schema', + 'sample_table', + 'sample_connection', + 'sample_schema', + 'sample_table', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + diff --git a/docs/client/operations/data_sources.md b/docs/client/operations/data_sources.md index 1f939a5764..cac4610275 100644 --- a/docs/client/operations/data_sources.md +++ b/docs/client/operations/data_sources.md @@ -469,7 +469,7 @@ http://localhost:8888/api/datasource/testconnection -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"connection_name\":\"sample_connection\",\"parallel_jobs_limit\":4,\"provider_type\":\"postgresql\",\"postgresql\":{\"host\":\"localhost\",\"port\":\"5432\",\"database\":\"db\",\"user\":\"PASSWD\",\"sslmode\":\"disable\"},\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"partitioned\"},\"collect_statistics_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"columnNames\":[]},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":false,\"deleteIncidents\":false},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" + "{\"connection_name\":\"sample_connection\",\"parallel_jobs_limit\":4,\"provider_type\":\"postgresql\",\"postgresql\":{\"host\":\"localhost\",\"port\":\"5432\",\"database\":\"db\",\"user\":\"PASSWD\",\"sslmode\":\"disable\"},\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"checkType\":\"partitioned\"},\"collect_statistics_job_template\":{\"connection\":\"sample_connection\",\"enabled\":true,\"columnNames\":[]},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"deleteChecksConfiguration\":false},\"advanced_properties\":{},\"can_edit\":false,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" ``` @@ -545,9 +545,13 @@ http://localhost:8888/api/datasource/testconnection delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -633,9 +637,13 @@ http://localhost:8888/api/datasource/testconnection delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -724,9 +732,13 @@ http://localhost:8888/api/datasource/testconnection delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -815,9 +827,13 @@ http://localhost:8888/api/datasource/testconnection delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, diff --git a/docs/client/operations/environment.md b/docs/client/operations/environment.md index cba9e84916..54b375bba8 100644 --- a/docs/client/operations/environment.md +++ b/docs/client/operations/environment.md @@ -253,7 +253,8 @@ http://localhost:8888/api/environment/profile "can_compare_tables" : false, "can_manage_users" : false, "can_manage_and_view_shared_credentials" : false, - "can_change_own_password" : false + "can_change_own_password" : false, + "can_use_data_domains" : false } ``` @@ -299,7 +300,8 @@ http://localhost:8888/api/environment/profile can_compare_tables=False, can_manage_users=False, can_manage_and_view_shared_credentials=False, - can_change_own_password=False + can_change_own_password=False, + can_use_data_domains=False ) ``` @@ -346,7 +348,8 @@ http://localhost:8888/api/environment/profile can_compare_tables=False, can_manage_users=False, can_manage_and_view_shared_credentials=False, - can_change_own_password=False + can_change_own_password=False, + can_use_data_domains=False ) ``` @@ -396,7 +399,8 @@ http://localhost:8888/api/environment/profile can_compare_tables=False, can_manage_users=False, can_manage_and_view_shared_credentials=False, - can_change_own_password=False + can_change_own_password=False, + can_use_data_domains=False ) ``` @@ -446,7 +450,8 @@ http://localhost:8888/api/environment/profile can_compare_tables=False, can_manage_users=False, can_manage_and_view_shared_credentials=False, - can_change_own_password=False + can_change_own_password=False, + can_use_data_domains=False ) ``` diff --git a/docs/client/operations/index.md b/docs/client/operations/index.md index 5f512b9d1f..14479545f7 100644 --- a/docs/client/operations/index.md +++ b/docs/client/operations/index.md @@ -45,6 +45,32 @@ Data quality check definition management operations for adding/removing/changing |[`update_check`](./checks.md#update_check)|PUT|Updates an existing check, making a custom check definition if it is not present| +## column_quality_policies +Operations for managing the configuration of data quality policies at a column level. Policies are the default configuration of data quality checks for columns matching a pattern. + +| Operation name | HTTP call | Description                     | +|----------------|------|---------------------------------| +|[`copy_from_column_quality_policy`](./column_quality_policies.md#copy_from_column_quality_policy)|POST|Creates (adds) a copy of an existing default column-level checks pattern configuration, under a new name.| +|[`create_column_quality_policy`](./column_quality_policies.md#create_column_quality_policy)|POST|Creates (adds) a new default column-level checks pattern configuration by saving a full specification object.| +|[`create_column_quality_policy_target`](./column_quality_policies.md#create_column_quality_policy_target)|POST|Creates (adds) a new default column-level checks pattern configuration.| +|[`delete_column_quality_policy`](./column_quality_policies.md#delete_column_quality_policy)|DELETE|Deletes a default column-level checks pattern| +|[`get_column_quality_policies`](./column_quality_policies.md#get_column_quality_policies)|GET|Returns a flat list of all column-level default check patterns configured for this instance. Default checks are applied on columns dynamically.| +|[`get_column_quality_policy`](./column_quality_policies.md#get_column_quality_policy)|GET|Returns a default checks pattern definition as a full specification object| +|[`get_column_quality_policy_target`](./column_quality_policies.md#get_column_quality_policy_target)|GET|Returns a default checks pattern definition| +|[`get_monitoring_daily_column_quality_policy`](./column_quality_policies.md#get_monitoring_daily_column_quality_policy)|GET|Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a column level.| +|[`get_monitoring_monthly_column_quality_policy`](./column_quality_policies.md#get_monitoring_monthly_column_quality_policy)|GET|Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a column level.| +|[`get_partitioned_daily_column_quality_policy`](./column_quality_policies.md#get_partitioned_daily_column_quality_policy)|GET|Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a column level.| +|[`get_partitioned_monthly_column_quality_policy`](./column_quality_policies.md#get_partitioned_monthly_column_quality_policy)|GET|Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a column level.| +|[`get_profiling_column_quality_policy`](./column_quality_policies.md#get_profiling_column_quality_policy)|GET|Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a column level.| +|[`update_column_quality_policy`](./column_quality_policies.md#update_column_quality_policy)|PUT|Updates an default column-level checks pattern by saving a full specification object| +|[`update_column_quality_policy_target`](./column_quality_policies.md#update_column_quality_policy_target)|PUT|Updates an default column-level checks pattern, changing only the target object| +|[`update_monitoring_daily_column_quality_policy`](./column_quality_policies.md#update_monitoring_daily_column_quality_policy)|PUT|New configuration of the default daily monitoring checks on a column level. These checks will be applied to columns.| +|[`update_monitoring_monthly_column_quality_policy`](./column_quality_policies.md#update_monitoring_monthly_column_quality_policy)|PUT|New configuration of the default monthly monitoring checks on a column level. These checks will be applied to columns.| +|[`update_partitioned_daily_column_quality_policy`](./column_quality_policies.md#update_partitioned_daily_column_quality_policy)|PUT|New configuration of the default daily partitioned checks on a column level. These checks will be applied to columns.| +|[`update_partitioned_monthly_column_quality_policy`](./column_quality_policies.md#update_partitioned_monthly_column_quality_policy)|PUT|New configuration of the default monthly partitioned checks on a column level. These checks will be applied to columns.| +|[`update_profiling_column_quality_policy`](./column_quality_policies.md#update_profiling_column_quality_policy)|PUT|New configuration of the default profiling checks on a column level. These checks will be applied to columns.| + + ## columns Operations related to manage the metadata of columns, and managing the configuration of column-level data quality checks. @@ -128,6 +154,18 @@ Operations for retrieving the list of data quality dashboards supported by DQOps |[`get_dashboard_level_5`](./dashboards.md#get_dashboard_level_5)|GET|Returns a single dashboard in the tree of folders with a temporary authenticated url| +## data_domains +Data domain management API to create different data domains. + +| Operation name | HTTP call | Description                     | +|----------------|------|---------------------------------| +|[`create_data_domain`](./data_domains.md#create_data_domain)|POST|Creates a new data domain given a data domain display name.| +|[`delete_data_domain`](./data_domains.md#delete_data_domain)|DELETE|Deletes a data domain. The domain is deleted in the DQOps SaaS cloud and locally.| +|[`get_local_data_domains`](./data_domains.md#get_local_data_domains)|GET|Returns a list of local data domains that this instance is maintaining. Data domains are supported only in an ENTERPRISE versions of DQOps.| +|[`switch_to_data_domain`](./data_domains.md#switch_to_data_domain)|GET|Switches to a different data domain. This operation sends a special cookie and redirects the user to the home screen.| +|[`synchronize_data_domains`](./data_domains.md#synchronize_data_domains)|PATCH|Synchronizes the domains in the SaaS cloud to this instance. All data domains will be created locally.| + + ## data_grouping_configurations Operations for managing the configuration of data groupings on a table level in DQOps. @@ -141,6 +179,18 @@ Operations for managing the configuration of data groupings on a table level in |[`update_table_grouping_configuration`](./data_grouping_configurations.md#update_table_grouping_configuration)|PUT|Updates a data grouping configuration according to the provided model| +## data_lineage +Operations related to managing and inspecting table and column lineage. + +| Operation name | HTTP call | Description                     | +|----------------|------|---------------------------------| +|[`create_table_source_table`](./data_lineage.md#create_table_source_table)|POST|Creates a new source table of the table's data lineage.| +|[`delete_table_source_table`](./data_lineage.md#delete_table_source_table)|DELETE|Deletes a specific data lineage source table of the given table.| +|[`get_table_source_table`](./data_lineage.md#get_table_source_table)|GET|Reads a specific data lineage source table defined on a target tale.| +|[`get_table_source_tables`](./data_lineage.md#get_table_source_tables)|GET|Returns a list of source tables on the data lineage that are sources of the given table.| +|[`update_table_source_table`](./data_lineage.md#update_table_source_table)|PUT|Update a specific data lineage source table using a new model.| + + ## data_sources Rest API controller that operates on data sources that are not yet imported, testing connections or retrieving the metadata (schemas and tables). @@ -151,58 +201,6 @@ Rest API controller that operates on data sources that are not yet imported, tes |[`test_connection`](./data_sources.md#test_connection)|POST|Checks if the given remote connection can be opened and if the credentials are valid| -## default_column_check_patterns -Operations for managing the configuration of the default column-level checks for columns matching a pattern. - -| Operation name | HTTP call | Description                     | -|----------------|------|---------------------------------| -|[`copy_from_default_column_checks_pattern`](./default_column_check_patterns.md#copy_from_default_column_checks_pattern)|POST|Creates (adds) a copy of an existing default column-level checks pattern configuration, under a new name.| -|[`create_default_column_checks_pattern`](./default_column_check_patterns.md#create_default_column_checks_pattern)|POST|Creates (adds) a new default column-level checks pattern configuration by saving a full specification object.| -|[`create_default_column_checks_pattern_target`](./default_column_check_patterns.md#create_default_column_checks_pattern_target)|POST|Creates (adds) a new default column-level checks pattern configuration.| -|[`delete_default_column_checks_pattern`](./default_column_check_patterns.md#delete_default_column_checks_pattern)|DELETE|Deletes a default column-level checks pattern| -|[`get_all_default_column_checks_patterns`](./default_column_check_patterns.md#get_all_default_column_checks_patterns)|GET|Returns a flat list of all column-level default check patterns configured for this instance. Default checks are applied on columns dynamically.| -|[`get_default_column_checks_pattern`](./default_column_check_patterns.md#get_default_column_checks_pattern)|GET|Returns a default checks pattern definition as a full specification object| -|[`get_default_column_checks_pattern_target`](./default_column_check_patterns.md#get_default_column_checks_pattern_target)|GET|Returns a default checks pattern definition| -|[`get_default_monitoring_daily_column_checks_pattern`](./default_column_check_patterns.md#get_default_monitoring_daily_column_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a column level.| -|[`get_default_monitoring_monthly_column_checks_pattern`](./default_column_check_patterns.md#get_default_monitoring_monthly_column_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a column level.| -|[`get_default_partitioned_daily_column_checks_pattern`](./default_column_check_patterns.md#get_default_partitioned_daily_column_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a column level.| -|[`get_default_partitioned_monthly_column_checks_pattern`](./default_column_check_patterns.md#get_default_partitioned_monthly_column_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a column level.| -|[`get_default_profiling_column_checks_pattern`](./default_column_check_patterns.md#get_default_profiling_column_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a column level.| -|[`update_default_column_checks_pattern`](./default_column_check_patterns.md#update_default_column_checks_pattern)|PUT|Updates an default column-level checks pattern by saving a full specification object| -|[`update_default_column_checks_pattern_target`](./default_column_check_patterns.md#update_default_column_checks_pattern_target)|PUT|Updates an default column-level checks pattern, changing only the target object| -|[`update_default_monitoring_daily_column_checks_pattern`](./default_column_check_patterns.md#update_default_monitoring_daily_column_checks_pattern)|PUT|New configuration of the default daily monitoring checks on a column level. These checks will be applied to columns.| -|[`update_default_monitoring_monthly_column_checks_pattern`](./default_column_check_patterns.md#update_default_monitoring_monthly_column_checks_pattern)|PUT|New configuration of the default monthly monitoring checks on a column level. These checks will be applied to columns.| -|[`update_default_partitioned_daily_column_checks_pattern`](./default_column_check_patterns.md#update_default_partitioned_daily_column_checks_pattern)|PUT|New configuration of the default daily partitioned checks on a column level. These checks will be applied to columns.| -|[`update_default_partitioned_monthly_column_checks_pattern`](./default_column_check_patterns.md#update_default_partitioned_monthly_column_checks_pattern)|PUT|New configuration of the default monthly partitioned checks on a column level. These checks will be applied to columns.| -|[`update_default_profiling_column_checks_pattern`](./default_column_check_patterns.md#update_default_profiling_column_checks_pattern)|PUT|New configuration of the default profiling checks on a column level. These checks will be applied to columns.| - - -## default_table_check_patterns -Operations for managing the configuration of the default table-level checks for tables matching a pattern. - -| Operation name | HTTP call | Description                     | -|----------------|------|---------------------------------| -|[`copy_from_default_table_checks_pattern`](./default_table_check_patterns.md#copy_from_default_table_checks_pattern)|POST|Creates (adds) a copy of an existing default table-level checks pattern configuration, under a new name.| -|[`create_default_table_checks_pattern`](./default_table_check_patterns.md#create_default_table_checks_pattern)|POST|Creates (adds) a new default table-level checks pattern configuration by saving a full specification object.| -|[`create_default_table_checks_pattern_target`](./default_table_check_patterns.md#create_default_table_checks_pattern_target)|POST|Creates (adds) a new default table-level checks pattern configuration.| -|[`delete_default_table_checks_pattern`](./default_table_check_patterns.md#delete_default_table_checks_pattern)|DELETE|Deletes a default table-level checks pattern| -|[`get_all_default_table_checks_patterns`](./default_table_check_patterns.md#get_all_default_table_checks_patterns)|GET|Returns a flat list of all table-level default check patterns configured for this instance. Default checks are applied on tables dynamically.| -|[`get_default_monitoring_daily_table_checks_pattern`](./default_table_check_patterns.md#get_default_monitoring_daily_table_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a table level.| -|[`get_default_monitoring_monthly_table_checks_pattern`](./default_table_check_patterns.md#get_default_monitoring_monthly_table_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a table level.| -|[`get_default_partitioned_daily_table_checks_pattern`](./default_table_check_patterns.md#get_default_partitioned_daily_table_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a table level.| -|[`get_default_partitioned_monthly_table_checks_pattern`](./default_table_check_patterns.md#get_default_partitioned_monthly_table_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a table level.| -|[`get_default_profiling_table_checks_pattern`](./default_table_check_patterns.md#get_default_profiling_table_checks_pattern)|GET|Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a table level.| -|[`get_default_table_checks_pattern`](./default_table_check_patterns.md#get_default_table_checks_pattern)|GET|Returns a default checks pattern definition as a full specification object| -|[`get_default_table_checks_pattern_target`](./default_table_check_patterns.md#get_default_table_checks_pattern_target)|GET|Returns a default checks pattern definition| -|[`update_default_monitoring_daily_table_checks_pattern`](./default_table_check_patterns.md#update_default_monitoring_daily_table_checks_pattern)|PUT|New configuration of the default daily monitoring checks on a table level. These checks will be applied to tables.| -|[`update_default_monitoring_monthly_table_checks_pattern`](./default_table_check_patterns.md#update_default_monitoring_monthly_table_checks_pattern)|PUT|New configuration of the default monthly monitoring checks on a table level. These checks will be applied to tables.| -|[`update_default_partitioned_daily_table_checks_pattern`](./default_table_check_patterns.md#update_default_partitioned_daily_table_checks_pattern)|PUT|New configuration of the default daily partitioned checks on a table level. These checks will be applied to tables.| -|[`update_default_partitioned_monthly_table_checks_pattern`](./default_table_check_patterns.md#update_default_partitioned_monthly_table_checks_pattern)|PUT|New configuration of the default monthly partitioned checks on a table level. These checks will be applied to tables.| -|[`update_default_profiling_table_checks_pattern`](./default_table_check_patterns.md#update_default_profiling_table_checks_pattern)|PUT|New configuration of the default profiling checks on a table level. These checks will be applied to tables.| -|[`update_default_table_checks_pattern`](./default_table_check_patterns.md#update_default_table_checks_pattern)|PUT|Updates an default table-level checks pattern by saving a full specification object| -|[`update_default_table_checks_pattern_target`](./default_table_check_patterns.md#update_default_table_checks_pattern_target)|PUT|Updates an default table-level checks pattern, changing only the target object| - - ## defaults Default settings management for configuring the default data quality checks that are configured for all imported tables and columns. @@ -482,6 +480,32 @@ Operations for managing the configurations of table comparisons between tables o |[`update_table_comparison_profiling`](./table_comparisons.md#update_table_comparison_profiling)|PUT|Updates a table comparison profiling checks| +## table_quality_policies +Operations for managing the configuration of data quality policies at a table level. Policies are the default configuration of data quality checks for tables matching a pattern. + +| Operation name | HTTP call | Description                     | +|----------------|------|---------------------------------| +|[`copy_from_table_quality_policy`](./table_quality_policies.md#copy_from_table_quality_policy)|POST|Creates (adds) a copy of an existing default table-level checks pattern configuration (data quality policy) under a new name.| +|[`create_table_quality_policy_pattern`](./table_quality_policies.md#create_table_quality_policy_pattern)|POST|Creates (adds) a new default table-level checks pattern (data quality policy) configuration by saving a full specification object.| +|[`create_table_quality_policy_target`](./table_quality_policies.md#create_table_quality_policy_target)|POST|Creates (adds) a new default table-level checks pattern configuration (a table-level data quality policy).| +|[`delete_table_quality_policy`](./table_quality_policies.md#delete_table_quality_policy)|DELETE|Deletes a default table-level checks pattern (a data quality policy at a column level).| +|[`get_monitoring_daily_table_quality_policy`](./table_quality_policies.md#get_monitoring_daily_table_quality_policy)|GET|Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a table level.| +|[`get_monitoring_monthly_table_quality_policy`](./table_quality_policies.md#get_monitoring_monthly_table_quality_policy)|GET|Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a table level.| +|[`get_partitioned_daily_table_quality_policy`](./table_quality_policies.md#get_partitioned_daily_table_quality_policy)|GET|Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a table level.| +|[`get_partitioned_monthly_table_quality_policy`](./table_quality_policies.md#get_partitioned_monthly_table_quality_policy)|GET|Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a table level.| +|[`get_profiling_table_quality_policy`](./table_quality_policies.md#get_profiling_table_quality_policy)|GET|Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a table level.| +|[`get_table_quality_policies`](./table_quality_policies.md#get_table_quality_policies)|GET|Returns a flat list of all table-level default check patterns (data quality policies) configured for this instance. Default checks are applied on tables dynamically.| +|[`get_table_quality_policy`](./table_quality_policies.md#get_table_quality_policy)|GET|Returns a default table-level checks pattern (data quality policy) definition as a full specification object| +|[`get_table_quality_policy_target`](./table_quality_policies.md#get_table_quality_policy_target)|GET|Returns a default checks pattern definition (a data quality policy)| +|[`update_monitoring_daily_table_quality_policy`](./table_quality_policies.md#update_monitoring_daily_table_quality_policy)|PUT|New configuration of the default daily monitoring checks on a table level. These checks will be applied to tables.| +|[`update_monitoring_monthly_table_quality_policy`](./table_quality_policies.md#update_monitoring_monthly_table_quality_policy)|PUT|New configuration of the default monthly monitoring checks on a table level. These checks will be applied to tables.| +|[`update_partitioned_daily_table_quality_policy`](./table_quality_policies.md#update_partitioned_daily_table_quality_policy)|PUT|New configuration of the default daily partitioned checks on a table level. These checks will be applied to tables.| +|[`update_partitioned_monthly_table_quality_policy`](./table_quality_policies.md#update_partitioned_monthly_table_quality_policy)|PUT|New configuration of the default monthly partitioned checks on a table level. These checks will be applied to tables.| +|[`update_profiling_table_quality_policy`](./table_quality_policies.md#update_profiling_table_quality_policy)|PUT|New configuration of the default profiling checks on a table level. These checks will be applied to tables.| +|[`update_table_quality_policy`](./table_quality_policies.md#update_table_quality_policy)|PUT|Updates an default table-level checks pattern (data quality policy) by saving a full specification object| +|[`update_table_quality_policy_target`](./table_quality_policies.md#update_table_quality_policy_target)|PUT|Updates an default table-level checks pattern (data quality policy), changing only the target object| + + ## tables Operations related to manage the metadata of imported tables, and managing the configuration of table-level data quality checks. diff --git a/docs/client/operations/jobs.md b/docs/client/operations/jobs.md index a8191d967c..c3e757dc7a 100644 --- a/docs/client/operations/jobs.md +++ b/docs/client/operations/jobs.md @@ -445,6 +445,8 @@ http://localhost:8888/api/jobs/collectstatistics/withgrouping | Property name | Description                     | Data type | Required | |---------------|---------------------------------|-----------|-----------------| |`job_business_key`|Optional job business key that is a user assigned unique job id, used to check the job status by looking up the job by a user assigned identifier, instead of the DQOps assigned job identifier.|*string*| | +|`samples_limit`|The limit of column samples to collect.|*long*| | +|`configure_table`|Configures timestamp and ID column on the table based on the results identified in statistics.|*boolean*| | |`wait`|Wait until the statistic collection job finishes to run, the default value is true (queue a background job and wait for the job to finish, up to waitTimeout seconds)|*boolean*| | |`wait_timeout`|The wait timeout in seconds, when the wait timeout elapses and the job is still running, only the job id is returned without the results. The default timeout is 120 seconds, but it can be reconfigured (see the 'dqo' cli command documentation).|*long*| | @@ -744,6 +746,8 @@ http://localhost:8888/api/jobs/collectstatistics/table | Property name | Description                     | Data type | Required | |---------------|---------------------------------|-----------|-----------------| |`job_business_key`|Optional job business key that is a user assigned unique job id, used to check the job status by looking up the job by a user assigned identifier, instead of the DQOps assigned job identifier.|*string*| | +|`configure_table`|Configures timestamp and ID column on the table based on the results identified in statistics.|*boolean*| | +|`samples_limit`|The limit of column samples to collect.|*long*| | |`wait`|Wait until the statistic collection job finishes to run, the default value is true (queue a background job and wait for the job to finish, up to waitTimeout seconds)|*boolean*| | |`wait_timeout`|The wait timeout in seconds, when the wait timeout elapses and the job is still running, only the job id is returned without the results. The default timeout is 120 seconds, but it can be reconfigured (see the 'dqo' cli command documentation).|*long*| | @@ -1069,7 +1073,7 @@ http://localhost:8888/api/jobs/deletestoreddata -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"columnNames\":[\"sample_column\"]}" + "{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"deleteChecksConfiguration\":false,\"columnNames\":[\"sample_column\"]}" ``` @@ -1106,11 +1110,12 @@ http://localhost:8888/api/jobs/deletestoreddata connection='sample_connection', full_table_name='sample_schema.sample_table', delete_errors=True, - delete_statistics=False, + delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, delete_incidents=True, + delete_checks_configuration=False, column_names=[ 'sample_column' ] @@ -1156,11 +1161,12 @@ http://localhost:8888/api/jobs/deletestoreddata connection='sample_connection', full_table_name='sample_schema.sample_table', delete_errors=True, - delete_statistics=False, + delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, delete_incidents=True, + delete_checks_configuration=False, column_names=[ 'sample_column' ] @@ -1209,11 +1215,12 @@ http://localhost:8888/api/jobs/deletestoreddata connection='sample_connection', full_table_name='sample_schema.sample_table', delete_errors=True, - delete_statistics=False, + delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, delete_incidents=True, + delete_checks_configuration=False, column_names=[ 'sample_column' ] @@ -1262,11 +1269,12 @@ http://localhost:8888/api/jobs/deletestoreddata connection='sample_connection', full_table_name='sample_schema.sample_table', delete_errors=True, - delete_statistics=False, + delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, delete_incidents=True, + delete_checks_configuration=False, column_names=[ 'sample_column' ] @@ -2207,7 +2215,10 @@ http://localhost:8888/api/jobs/importtables ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) ] ), @@ -2287,7 +2298,10 @@ http://localhost:8888/api/jobs/importtables ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) ] ), @@ -2370,7 +2384,10 @@ http://localhost:8888/api/jobs/importtables ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) ] ), @@ -2453,7 +2470,10 @@ http://localhost:8888/api/jobs/importtables ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) ] ), diff --git a/docs/client/operations/rule_mining.md b/docs/client/operations/rule_mining.md index 5442a21ca0..e0e6ed6539 100644 --- a/docs/client/operations/rule_mining.md +++ b/docs/client/operations/rule_mining.md @@ -709,7 +709,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"severity_level\":\"error\",\"copy_failed_profiling_checks\":false,\"copy_disabled_profiling_checks\":false,\"copy_profiling_checks\":true,\"propose_default_checks\":true,\"propose_minimum_row_count\":true,\"propose_column_count\":true,\"propose_timeliness_checks\":true,\"propose_nulls_checks\":true,\"propose_not_nulls_checks\":true,\"propose_text_values_data_type\":true,\"propose_column_exists\":true,\"propose_uniqueness_checks\":true,\"propose_numeric_ranges\":true,\"propose_percentile_ranges\":true,\"propose_text_length_ranges\":true,\"propose_word_count_ranges\":true,\"propose_values_in_set_checks\":true,\"values_in_set_treat_rare_values_as_invalid\":true,\"propose_top_values_checks\":true,\"propose_text_conversion_checks\":true,\"propose_bool_percent_checks\":true,\"propose_date_checks\":true,\"propose_standard_pattern_checks\":true,\"detect_regular_expressions\":true,\"propose_whitespace_checks\":true,\"apply_pii_checks\":true,\"propose_custom_checks\":true}" + "{\"severity_level\":\"error\",\"copy_failed_profiling_checks\":false,\"copy_disabled_profiling_checks\":false,\"copy_profiling_checks\":true,\"reconfigure_policy_enabled_checks\":true,\"propose_checks_from_statistics\":true,\"propose_minimum_row_count\":true,\"propose_column_count\":true,\"propose_timeliness_checks\":true,\"propose_nulls_checks\":true,\"propose_not_nulls_checks\":true,\"propose_text_values_data_type\":true,\"propose_column_exists\":true,\"propose_uniqueness_checks\":true,\"propose_numeric_ranges\":true,\"propose_percentile_ranges\":true,\"propose_text_length_ranges\":true,\"propose_word_count_ranges\":true,\"propose_values_in_set_checks\":true,\"values_in_set_treat_rare_values_as_invalid\":true,\"propose_top_values_checks\":true,\"propose_text_conversion_checks\":true,\"propose_bool_percent_checks\":true,\"propose_date_checks\":true,\"propose_standard_pattern_checks\":true,\"detect_regular_expressions\":true,\"propose_whitespace_checks\":true,\"apply_pii_checks\":true,\"propose_custom_checks\":true}" ``` @@ -747,7 +747,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -820,7 +821,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -896,7 +898,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -972,7 +975,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1080,7 +1084,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"severity_level\":\"error\",\"copy_failed_profiling_checks\":false,\"copy_disabled_profiling_checks\":false,\"copy_profiling_checks\":true,\"propose_default_checks\":true,\"propose_minimum_row_count\":true,\"propose_column_count\":true,\"propose_timeliness_checks\":true,\"propose_nulls_checks\":true,\"propose_not_nulls_checks\":true,\"propose_text_values_data_type\":true,\"propose_column_exists\":true,\"propose_uniqueness_checks\":true,\"propose_numeric_ranges\":true,\"propose_percentile_ranges\":true,\"propose_text_length_ranges\":true,\"propose_word_count_ranges\":true,\"propose_values_in_set_checks\":true,\"values_in_set_treat_rare_values_as_invalid\":true,\"propose_top_values_checks\":true,\"propose_text_conversion_checks\":true,\"propose_bool_percent_checks\":true,\"propose_date_checks\":true,\"propose_standard_pattern_checks\":true,\"detect_regular_expressions\":true,\"propose_whitespace_checks\":true,\"apply_pii_checks\":true,\"propose_custom_checks\":true}" + "{\"severity_level\":\"error\",\"copy_failed_profiling_checks\":false,\"copy_disabled_profiling_checks\":false,\"copy_profiling_checks\":true,\"reconfigure_policy_enabled_checks\":true,\"propose_checks_from_statistics\":true,\"propose_minimum_row_count\":true,\"propose_column_count\":true,\"propose_timeliness_checks\":true,\"propose_nulls_checks\":true,\"propose_not_nulls_checks\":true,\"propose_text_values_data_type\":true,\"propose_column_exists\":true,\"propose_uniqueness_checks\":true,\"propose_numeric_ranges\":true,\"propose_percentile_ranges\":true,\"propose_text_length_ranges\":true,\"propose_word_count_ranges\":true,\"propose_values_in_set_checks\":true,\"values_in_set_treat_rare_values_as_invalid\":true,\"propose_top_values_checks\":true,\"propose_text_conversion_checks\":true,\"propose_bool_percent_checks\":true,\"propose_date_checks\":true,\"propose_standard_pattern_checks\":true,\"detect_regular_expressions\":true,\"propose_whitespace_checks\":true,\"apply_pii_checks\":true,\"propose_custom_checks\":true}" ``` @@ -1118,7 +1122,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1191,7 +1196,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1267,7 +1273,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1343,7 +1350,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1450,7 +1458,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"severity_level\":\"error\",\"copy_failed_profiling_checks\":false,\"copy_disabled_profiling_checks\":false,\"copy_profiling_checks\":true,\"propose_default_checks\":true,\"propose_minimum_row_count\":true,\"propose_column_count\":true,\"propose_timeliness_checks\":true,\"propose_nulls_checks\":true,\"propose_not_nulls_checks\":true,\"propose_text_values_data_type\":true,\"propose_column_exists\":true,\"propose_uniqueness_checks\":true,\"propose_numeric_ranges\":true,\"propose_percentile_ranges\":true,\"propose_text_length_ranges\":true,\"propose_word_count_ranges\":true,\"propose_values_in_set_checks\":true,\"values_in_set_treat_rare_values_as_invalid\":true,\"propose_top_values_checks\":true,\"propose_text_conversion_checks\":true,\"propose_bool_percent_checks\":true,\"propose_date_checks\":true,\"propose_standard_pattern_checks\":true,\"detect_regular_expressions\":true,\"propose_whitespace_checks\":true,\"apply_pii_checks\":true,\"propose_custom_checks\":true}" + "{\"severity_level\":\"error\",\"copy_failed_profiling_checks\":false,\"copy_disabled_profiling_checks\":false,\"copy_profiling_checks\":true,\"reconfigure_policy_enabled_checks\":true,\"propose_checks_from_statistics\":true,\"propose_minimum_row_count\":true,\"propose_column_count\":true,\"propose_timeliness_checks\":true,\"propose_nulls_checks\":true,\"propose_not_nulls_checks\":true,\"propose_text_values_data_type\":true,\"propose_column_exists\":true,\"propose_uniqueness_checks\":true,\"propose_numeric_ranges\":true,\"propose_percentile_ranges\":true,\"propose_text_length_ranges\":true,\"propose_word_count_ranges\":true,\"propose_values_in_set_checks\":true,\"values_in_set_treat_rare_values_as_invalid\":true,\"propose_top_values_checks\":true,\"propose_text_conversion_checks\":true,\"propose_bool_percent_checks\":true,\"propose_date_checks\":true,\"propose_standard_pattern_checks\":true,\"detect_regular_expressions\":true,\"propose_whitespace_checks\":true,\"apply_pii_checks\":true,\"propose_custom_checks\":true}" ``` @@ -1487,7 +1495,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1558,7 +1567,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1632,7 +1642,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, @@ -1706,7 +1717,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl copy_failed_profiling_checks=False, copy_disabled_profiling_checks=False, copy_profiling_checks=True, - propose_default_checks=True, + reconfigure_policy_enabled_checks=True, + propose_checks_from_statistics=True, propose_minimum_row_count=True, propose_column_count=True, propose_timeliness_checks=True, diff --git a/docs/client/operations/search.md b/docs/client/operations/search.md index b9ced6fbc4..66a03a0f2b 100644 --- a/docs/client/operations/search.md +++ b/docs/client/operations/search.md @@ -76,6 +76,7 @@ http://localhost:8888/api/search/columns "nullable" : false, "length" : 256 }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -94,6 +95,7 @@ http://localhost:8888/api/search/columns "nullable" : false, "length" : 256 }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -112,6 +114,7 @@ http://localhost:8888/api/search/columns "nullable" : false, "length" : 256 }, + "advanced_properties" : { }, "can_edit" : false, "can_collect_statistics" : true, "can_run_checks" : true, @@ -164,6 +167,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -187,6 +193,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -210,6 +219,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -264,6 +276,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -287,6 +302,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -310,6 +328,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -367,6 +388,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -390,6 +414,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -413,6 +440,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -470,6 +500,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -493,6 +526,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -516,6 +552,9 @@ http://localhost:8888/api/search/columns nullable=False, length=256 ), + advanced_properties={ + + }, can_edit=False, can_collect_statistics=True, can_run_checks=True, @@ -625,9 +664,11 @@ http://localhost:8888/api/search/tables "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : true, "can_collect_statistics" : true, "can_run_checks" : true, @@ -673,9 +714,11 @@ http://localhost:8888/api/search/tables "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : true, "can_collect_statistics" : true, "can_run_checks" : true, @@ -721,9 +764,11 @@ http://localhost:8888/api/search/tables "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : true, "can_collect_statistics" : true, "can_run_checks" : true, @@ -803,9 +848,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -856,9 +905,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -909,9 +962,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -993,9 +1050,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1046,9 +1107,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1099,9 +1164,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1186,9 +1255,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1239,9 +1312,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1292,9 +1369,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1379,9 +1460,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1432,9 +1517,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1485,9 +1574,13 @@ http://localhost:8888/api/search/tables delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, diff --git a/docs/client/operations/table_comparisons.md b/docs/client/operations/table_comparisons.md index 75783940ca..4147906490 100644 --- a/docs/client/operations/table_comparisons.md +++ b/docs/client/operations/table_comparisons.md @@ -285,7 +285,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -325,10 +325,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -350,7 +347,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -407,10 +405,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -432,7 +427,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -492,10 +488,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -517,7 +510,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -577,10 +571,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -602,7 +593,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -672,7 +664,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -712,10 +704,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -737,7 +726,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -794,10 +784,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -819,7 +806,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -879,10 +867,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -904,7 +889,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -964,10 +950,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -989,7 +972,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1059,7 +1043,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -1099,10 +1083,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1124,7 +1105,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1181,10 +1163,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1206,7 +1185,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1266,10 +1246,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1291,7 +1268,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1351,10 +1329,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1376,7 +1351,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1446,7 +1422,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -1486,10 +1462,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1511,7 +1484,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1568,10 +1542,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1593,7 +1564,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1653,10 +1625,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1678,7 +1647,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1738,10 +1708,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1763,7 +1730,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1833,7 +1801,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -1873,10 +1841,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1898,7 +1863,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -1955,10 +1921,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -1980,7 +1943,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -2040,10 +2004,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -2065,7 +2026,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -2125,10 +2087,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -2150,7 +2109,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3074,7 +3034,6 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, "grouping_columns" : [ ], "default_compare_thresholds" : { - "warning_difference_percent" : 0.0, "error_difference_percent" : 1.0 }, "supports_compare_column_count" : true, @@ -3096,7 +3055,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteCheckResults" : true, "deleteSensorReadouts" : true, "deleteErrorSamples" : true, - "deleteIncidents" : true, + "deleteIncidents" : false, + "deleteChecksConfiguration" : false, "checkCategory" : "comparisons", "tableComparisonName" : "sample_table_comparison", "checkType" : "monitoring", @@ -3152,10 +3112,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3177,7 +3134,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3234,10 +3192,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3259,7 +3214,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3319,10 +3275,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3344,7 +3297,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3404,10 +3358,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3429,7 +3380,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3512,7 +3464,6 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, "grouping_columns" : [ ], "default_compare_thresholds" : { - "warning_difference_percent" : 0.0, "error_difference_percent" : 1.0 }, "supports_compare_column_count" : true, @@ -3534,7 +3485,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteCheckResults" : true, "deleteSensorReadouts" : true, "deleteErrorSamples" : true, - "deleteIncidents" : true, + "deleteIncidents" : false, + "deleteChecksConfiguration" : false, "checkCategory" : "comparisons", "tableComparisonName" : "sample_table_comparison", "checkType" : "monitoring", @@ -3590,10 +3542,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3615,7 +3564,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3672,10 +3622,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3697,7 +3644,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3757,10 +3705,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3782,7 +3727,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3842,10 +3788,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -3867,7 +3810,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -3950,7 +3894,6 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, "grouping_columns" : [ ], "default_compare_thresholds" : { - "warning_difference_percent" : 0.0, "error_difference_percent" : 1.0 }, "supports_compare_column_count" : true, @@ -3972,7 +3915,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteCheckResults" : true, "deleteSensorReadouts" : true, "deleteErrorSamples" : true, - "deleteIncidents" : true, + "deleteIncidents" : false, + "deleteChecksConfiguration" : false, "checkCategory" : "comparisons", "tableComparisonName" : "sample_table_comparison", "checkType" : "monitoring", @@ -4028,10 +3972,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4053,7 +3994,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4110,10 +4052,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4135,7 +4074,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4195,10 +4135,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4220,7 +4157,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4280,10 +4218,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4305,7 +4240,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4388,7 +4324,6 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, "grouping_columns" : [ ], "default_compare_thresholds" : { - "warning_difference_percent" : 0.0, "error_difference_percent" : 1.0 }, "supports_compare_column_count" : true, @@ -4410,7 +4345,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteCheckResults" : true, "deleteSensorReadouts" : true, "deleteErrorSamples" : true, - "deleteIncidents" : true, + "deleteIncidents" : false, + "deleteChecksConfiguration" : false, "checkCategory" : "comparisons", "tableComparisonName" : "sample_table_comparison", "checkType" : "monitoring", @@ -4466,10 +4402,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4491,7 +4424,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4548,10 +4482,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4573,7 +4504,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4633,10 +4565,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4658,7 +4587,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4718,10 +4648,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4743,7 +4670,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4826,7 +4754,6 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl }, "grouping_columns" : [ ], "default_compare_thresholds" : { - "warning_difference_percent" : 0.0, "error_difference_percent" : 1.0 }, "supports_compare_column_count" : true, @@ -4848,7 +4775,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteCheckResults" : true, "deleteSensorReadouts" : true, "deleteErrorSamples" : true, - "deleteIncidents" : true, + "deleteIncidents" : false, + "deleteChecksConfiguration" : false, "checkCategory" : "comparisons", "tableComparisonName" : "sample_table_comparison", "checkType" : "monitoring", @@ -4904,10 +4832,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -4929,7 +4854,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -4986,10 +4912,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5011,7 +4934,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5071,10 +4995,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5096,7 +5017,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5156,10 +5078,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5181,7 +5100,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5484,7 +5404,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -5524,10 +5444,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5549,7 +5466,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5607,10 +5525,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5632,7 +5547,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5693,10 +5609,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5718,7 +5631,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5779,10 +5693,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5804,7 +5715,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5876,7 +5788,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -5916,10 +5828,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -5941,7 +5850,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -5999,10 +5909,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6024,7 +5931,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6085,10 +5993,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6110,7 +6015,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6171,10 +6077,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6196,7 +6099,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6268,7 +6172,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -6308,10 +6212,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6333,7 +6234,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6391,10 +6293,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6416,7 +6315,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6477,10 +6377,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6502,7 +6399,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6563,10 +6461,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6588,7 +6483,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6660,7 +6556,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -6700,10 +6596,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6725,7 +6618,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6783,10 +6677,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6808,7 +6699,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6869,10 +6761,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6894,7 +6783,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -6955,10 +6845,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -6980,7 +6867,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -7052,7 +6940,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"warning_difference_percent\":0.0,\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" + "{\"table_comparison_configuration_name\":\"sample_table_comparison\",\"compared_connection\":\"unknown\",\"compared_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"reference_connection\":\"sample_connection\",\"reference_table\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"grouping_columns\":[],\"default_compare_thresholds\":{\"error_difference_percent\":1.0},\"supports_compare_column_count\":true,\"columns\":[],\"compare_table_run_checks_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\",\"timeScale\":\"daily\",\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\"},\"compare_table_clean_data_job_template\":{\"connection\":\"unknown\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":false,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":false,\"deleteChecksConfiguration\":false,\"checkCategory\":\"comparisons\",\"tableComparisonName\":\"sample_table_comparison\",\"checkType\":\"monitoring\",\"timeGradient\":\"day\"},\"can_edit\":true,\"can_run_compare_checks\":true,\"can_delete_data\":true}" ``` @@ -7092,10 +6980,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -7117,7 +7002,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -7175,10 +7061,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -7200,7 +7083,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -7261,10 +7145,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -7286,7 +7167,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', @@ -7347,10 +7229,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl grouping_columns=[ ], - default_compare_thresholds=CompareThresholdsModel( - warning_difference_percent=0.0, - error_difference_percent=1.0 - ), + default_compare_thresholds=CompareThresholdsModel(error_difference_percent=1.0), supports_compare_column_count=True, columns=[ @@ -7372,7 +7251,8 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_check_results=True, delete_sensor_readouts=True, delete_error_samples=True, - delete_incidents=True, + delete_incidents=False, + delete_checks_configuration=False, check_category='comparisons', table_comparison_name='sample_table_comparison', check_type='monitoring', diff --git a/docs/client/operations/table_quality_policies.md b/docs/client/operations/table_quality_policies.md new file mode 100644 index 0000000000..a983ddae01 --- /dev/null +++ b/docs/client/operations/table_quality_policies.md @@ -0,0 +1,5390 @@ +--- +title: DQOps REST API table_quality_policies operations +--- +# DQOps REST API table_quality_policies operations +Operations for managing the configuration of data quality policies at a table level. Policies are the default configuration of data quality checks for tables matching a pattern. + + +___ +## copy_from_table_quality_policy +Creates (adds) a copy of an existing default table-level checks pattern configuration (data quality policy) under a new name. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/copy_from_table_quality_policy.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/policies/checks/table/{targetPatternName}/copyfrom/{sourcePatternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`target_pattern_name`|Target pattern name|*string*|:material-check-bold:| +|`source_pattern_name`|Source pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/policies/checks/table/default/copyfrom/default^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import copy_from_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = copy_from_table_quality_policy.sync( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import copy_from_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = await copy_from_table_quality_policy.asyncio( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import copy_from_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = copy_from_table_quality_policy.sync( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import copy_from_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = await copy_from_table_quality_policy.asyncio( + 'default', + 'default', + client=dqops_client + ) + + ``` + + + + + +___ +## create_table_quality_policy_pattern +Creates (adds) a new default table-level checks pattern (data quality policy) configuration by saving a full specification object. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_pattern.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/policies/checks/table/{patternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model|*[TableQualityPolicyModel](../models/table_quality_policies.md#tablequalitypolicymodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/policies/checks/table/default^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"default\",\"policy_spec\":{\"priority\":1000,\"monitoring_checks\":{\"daily\":{\"volume\":{\"daily_row_count\":{\"warning\":{\"min_count\":1}}}}}},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_pattern + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = create_table_quality_policy_pattern.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_pattern + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await create_table_quality_policy_pattern.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_pattern + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = create_table_quality_policy_pattern.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_pattern + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await create_table_quality_policy_pattern.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## create_table_quality_policy_target +Creates (adds) a new default table-level checks pattern configuration (a table-level data quality policy). + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/create_table_quality_policy_target.py) to see the source code on GitHub. + + +**POST** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/target +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model with only the target filters|*[TableQualityPolicyListModel](../models/table_quality_policies.md#tablequalitypolicylistmodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X POST http://localhost:8888/api/policies/checks/table/default/target^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"default\",\"priority\":100,\"disabled\":false,\"target_table\":{\"connection\":\"dwh\",\"schema\":\"public\",\"table\":\"fact_*\"},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = create_table_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = await create_table_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = create_table_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import create_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = await create_table_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## delete_table_quality_policy +Deletes a default table-level checks pattern (a data quality policy at a column level). + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/delete_table_quality_policy.py) to see the source code on GitHub. + + +**DELETE** +``` +http://localhost:8888/api/policies/checks/table/{patternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X DELETE http://localhost:8888/api/policies/checks/table/default^ + -H "Accept: application/json" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import delete_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = delete_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import delete_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + call_result = await delete_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import delete_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = delete_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import delete_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + call_result = await delete_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + + + +___ +## get_monitoring_daily_table_quality_policy +Returns UI model to show and edit the default configuration of the daily monitoring checks that are configured for a check pattern on a table level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_monitoring_daily_table_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/monitoring/daily +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table/default/monitoring/daily^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_daily_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_daily_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_daily_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_daily_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_daily_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_daily_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_monitoring_monthly_table_quality_policy +Returns UI model to show and edit the default configuration of the monthly monitoring checks that are configured for a check pattern on a table level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_monitoring_monthly_table_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/monitoring/monthly +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table/default/monitoring/monthly^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_monthly_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_monthly_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_monthly_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_monthly_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_monitoring_monthly_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_monitoring_monthly_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_monitoring_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_partitioned_daily_table_quality_policy +Returns UI model to show and edit the default configuration of the daily partitioned checks that are configured for a check pattern on a table level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_partitioned_daily_table_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/partitioned/daily +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table/default/partitioned/daily^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_daily_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_daily_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_daily_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_daily_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_daily_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_daily_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_partitioned_monthly_table_quality_policy +Returns UI model to show and edit the default configuration of the monthly partitioned checks that are configured for a check pattern on a table level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_partitioned_monthly_table_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/partitioned/monthly +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table/default/partitioned/monthly^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_monthly_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_monthly_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_monthly_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_monthly_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_partitioned_monthly_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_partitioned_monthly_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_partitioned_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_profiling_table_quality_policy +Returns UI model to show and edit the default configuration of the profiling checks that are configured for a check pattern on a table level. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_profiling_table_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/profiling +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`check_container_model`](../models/common.md#checkcontainermodel)||*[CheckContainerModel](../models/common.md#checkcontainermodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table/default/profiling^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "categories" : [ { + "category" : "sample_category", + "help_text" : "Sample help text", + "checks" : [ { + "check_name" : "sample_check", + "help_text" : "Sample help text", + "sensor_parameters" : [ ], + "sensor_name" : "sample_target/sample_category/table/volume/row_count", + "quality_dimension" : "sample_quality_dimension", + "supports_error_sampling" : false, + "supports_grouping" : false, + "default_severity" : "error", + "disabled" : false, + "exclude_from_kpi" : false, + "include_in_sla" : false, + "configured" : false, + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } ] + } ], + "can_edit" : false, + "can_run_checks" : false, + "can_delete_data" : false + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_profiling_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_profiling_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_profiling_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_profiling_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_profiling_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_profiling_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_profiling_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_profiling_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ``` + + + + + + +___ +## get_table_quality_policies +Returns a flat list of all table-level default check patterns (data quality policies) configured for this instance. Default checks are applied on tables dynamically. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policies.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|`table_quality_policy_list_model`||*List[[TableQualityPolicyListModel](../models/table_quality_policies.md#tablequalitypolicylistmodel)]*| + + + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + [ { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_table" : { + "connection" : "dwh", + "schema" : "public", + "table" : "fact_*" + }, + "can_edit" : true + }, { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_table" : { + "connection" : "dwh", + "schema" : "public", + "table" : "fact_*" + }, + "can_edit" : true + }, { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_table" : { + "connection" : "dwh", + "schema" : "public", + "table" : "fact_*" + }, + "can_edit" : true + } ] + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policies + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_table_quality_policies.sync( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ] + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policies + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_table_quality_policies.asyncio( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ] + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policies + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_table_quality_policies.sync( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ] + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policies + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_table_quality_policies.asyncio( + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + [ + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ), + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ] + ``` + + + + + + +___ +## get_table_quality_policy +Returns a default table-level checks pattern (data quality policy) definition as a full specification object + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table/{patternName} +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`table_quality_policy_model`](../models/table_quality_policies.md#tablequalitypolicymodel)||*[TableQualityPolicyModel](../models/table_quality_policies.md#tablequalitypolicymodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Table pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table/default^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "policy_name" : "default", + "policy_spec" : { + "priority" : 1000, + "monitoring_checks" : { + "daily" : { + "volume" : { + "daily_row_count" : { + "warning" : { + "min_count" : 1 + } + } + } + } + } + }, + "can_edit" : true + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_table_quality_policy.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_table_quality_policy.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + ``` + + + + + + +___ +## get_table_quality_policy_target +Returns a default checks pattern definition (a data quality policy) + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/get_table_quality_policy_target.py) to see the source code on GitHub. + + +**GET** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/target +``` + +**Return value** + +| Property name | Description                     | Data type | +|---------------|---------------------------------|-----------| +|[`table_quality_policy_list_model`](../models/table_quality_policies.md#tablequalitypolicylistmodel)||*[TableQualityPolicyListModel](../models/table_quality_policies.md#tablequalitypolicylistmodel)*| + + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Table pattern name|*string*|:material-check-bold:| + + + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl http://localhost:8888/api/policies/checks/table/default/target^ + -H "Accept: application/json" + + ``` + + + ??? example "Expand to see the returned result" + + + ``` + { + "policy_name" : "default", + "priority" : 100, + "disabled" : false, + "target_table" : { + "connection" : "dwh", + "schema" : "public", + "table" : "fact_*" + }, + "can_edit" : true + } + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy_target + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = get_table_quality_policy_target.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ``` + + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy_target + + dqops_client = client.Client( + 'http://localhost:8888/', + raise_on_unexpected_status=True + ) + + call_result = await get_table_quality_policy_target.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy_target + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = get_table_quality_policy_target.sync( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ``` + + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import get_table_quality_policy_target + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token, + raise_on_unexpected_status=True + ) + + call_result = await get_table_quality_policy_target.asyncio( + 'default', + client=dqops_client + ) + + ``` + + + ??? example "Expand to see the returned result" + + ``` + TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + ``` + + + + + + +___ +## update_monitoring_daily_table_quality_policy +New configuration of the default daily monitoring checks on a table level. These checks will be applied to tables. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/update_monitoring_daily_table_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/monitoring/daily +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality daily monitoring checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/table/default/monitoring/daily^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_daily_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_daily_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_monitoring_monthly_table_quality_policy +New configuration of the default monthly monitoring checks on a table level. These checks will be applied to tables. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/update_monitoring_monthly_table_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/monitoring/monthly +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality monthly monitoring checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/table/default/monitoring/monthly^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_monthly_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_monitoring_monthly_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_monitoring_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_monitoring_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_partitioned_daily_table_quality_policy +New configuration of the default daily partitioned checks on a table level. These checks will be applied to tables. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/update_partitioned_daily_table_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/partitioned/daily +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality daily partitioned checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/table/default/partitioned/daily^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_daily_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_daily_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_daily_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_daily_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_partitioned_monthly_table_quality_policy +New configuration of the default monthly partitioned checks on a table level. These checks will be applied to tables. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/update_partitioned_monthly_table_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/partitioned/monthly +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality monthly partitioned checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/table/default/partitioned/monthly^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_monthly_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_partitioned_monthly_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_partitioned_monthly_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_partitioned_monthly_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_profiling_table_quality_policy +New configuration of the default profiling checks on a table level. These checks will be applied to tables. + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/update_profiling_table_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/profiling +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Model with the changes to be applied to the data quality profiling checks configuration|*[CheckContainerModel](../models/common.md#checkcontainermodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/table/default/profiling^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"categories\":[{\"category\":\"sample_category\",\"help_text\":\"Sample help text\",\"checks\":[{\"check_name\":\"sample_check\",\"help_text\":\"Sample help text\",\"sensor_parameters\":[],\"sensor_name\":\"sample_target/sample_category/table/volume/row_count\",\"quality_dimension\":\"sample_quality_dimension\",\"supports_error_sampling\":false,\"supports_grouping\":false,\"default_severity\":\"error\",\"disabled\":false,\"exclude_from_kpi\":false,\"include_in_sla\":false,\"configured\":false,\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}]}],\"can_edit\":false,\"can_run_checks\":false,\"can_delete_data\":false}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_profiling_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_profiling_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_profiling_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_profiling_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_profiling_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = update_profiling_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_profiling_table_quality_policy + from dqops.client.models import CheckContainerModel, \ + CheckModel, \ + DefaultRuleSeverityLevel, \ + FieldModel, \ + QualityCategoryModel + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = CheckContainerModel( + categories=[ + QualityCategoryModel( + category='sample_category', + help_text='Sample help text', + checks=[ + CheckModel( + check_name='sample_check', + help_text='Sample help text', + sensor_parameters=[ + + ], + sensor_name='sample_target/sample_category/table/volume/row_count', + quality_dimension='sample_quality_dimension', + supports_error_sampling=False, + supports_grouping=False, + standard=False, + default_check=False, + default_severity=DefaultRuleSeverityLevel.ERROR, + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + configured=False, + always_collect_error_samples=False, + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + ] + ) + ], + can_edit=False, + can_run_checks=False, + can_delete_data=False + ) + + call_result = await update_profiling_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_table_quality_policy +Updates an default table-level checks pattern (data quality policy) by saving a full specification object + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/table/{patternName} +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model|*[TableQualityPolicyModel](../models/table_quality_policies.md#tablequalitypolicymodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/table/default^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"default\",\"policy_spec\":{\"priority\":1000,\"monitoring_checks\":{\"daily\":{\"volume\":{\"daily_row_count\":{\"warning\":{\"min_count\":1}}}}}},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = update_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await update_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = update_table_quality_policy.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy + from dqops.client.models import MinCountRule1ParametersSpec, \ + TableMonitoringCheckCategoriesSpec, \ + TablePartitionedCheckCategoriesSpec, \ + TableProfilingCheckCategoriesSpec, \ + TableQualityPolicyModel, \ + TableQualityPolicySpec, \ + TableRowCountCheckSpec, \ + TableVolumeProfilingChecksSpec, \ + TableVolumeRowCountSensorParametersSpec, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyModel( + policy_name='default', + policy_spec=TableQualityPolicySpec( + priority=1000, + disabled=False, + target=TargetTablePatternSpec(), + profiling_checks=TableProfilingCheckCategoriesSpec(comparisons=TableComparisonProfilingChecksSpecMap()), + monitoring_checks=TableMonitoringCheckCategoriesSpec( + daily=TableDailyMonitoringCheckCategoriesSpec( + volume=TableVolumeDailyMonitoringChecksSpec( + daily_row_count=TableRowCountCheckSpec( + parameters=TableVolumeRowCountSensorParametersSpec(), + warning=MinCountRule1ParametersSpec(min_count=1), + disabled=False, + exclude_from_kpi=False, + include_in_sla=False, + always_collect_error_samples=False + ) + ), + comparisons=TableComparisonDailyMonitoringChecksSpecMap() + ) + ), + partitioned_checks=TablePartitionedCheckCategoriesSpec() + ), + can_edit=True + ) + + call_result = await update_table_quality_policy.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + +___ +## update_table_quality_policy_target +Updates an default table-level checks pattern (data quality policy), changing only the target object + +Follow the [link](https://github.com/dqops/dqo/blob/develop/distribution/python/dqops/client/api/table_quality_policies/update_table_quality_policy_target.py) to see the source code on GitHub. + + +**PUT** +``` +http://localhost:8888/api/policies/checks/table/{patternName}/target +``` + + + +**Parameters of this method are described below** + +| Property name | Description                     | Data type | Required | +|---------------|---------------------------------|-----------|-----------------| +|`pattern_name`|Pattern name|*string*|:material-check-bold:| + + + + +**Request body** + +| Description                     | Data type | Required | +|---------------------------------|-----------|-----------------| +|Default checks pattern model|*[TableQualityPolicyListModel](../models/table_quality_policies.md#tablequalitypolicylistmodel)*| | + + + + +**Usage examples** + + +=== "curl" + **Execution** + + ```bash + curl -X PUT http://localhost:8888/api/policies/checks/table/default/target^ + -H "Accept: application/json"^ + -H "Content-Type: application/json"^ + -d^ + "{\"policy_name\":\"default\",\"priority\":100,\"disabled\":false,\"target_table\":{\"connection\":\"dwh\",\"schema\":\"public\",\"table\":\"fact_*\"},\"can_edit\":true}" + + ``` + + + + +=== "Python sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = update_table_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + dqops_client = client.Client( + 'http://localhost:8888/' + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = await update_table_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth sync client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = update_table_quality_policy_target.sync( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + +=== "Python auth async client" + **Execution** + + ```python + from dqops import client + from dqops.client.api.table_quality_policies import update_table_quality_policy_target + from dqops.client.models import TableQualityPolicyListModel, \ + TargetTablePatternSpec + + token = 's4mp13_4u7h_70k3n' + + dqops_client = client.AuthenticatedClient( + 'http://localhost:8888/', + token=token + ) + + request_body = TableQualityPolicyListModel( + policy_name='default', + priority=100, + disabled=False, + target_table=TargetTablePatternSpec( + connection='dwh', + schema='public', + table='fact_*' + ), + can_edit=True + ) + + call_result = await update_table_quality_policy_target.asyncio( + 'default', + client=dqops_client, + json_body=request_body + ) + + ``` + + + + + diff --git a/docs/client/operations/tables.md b/docs/client/operations/tables.md index d8d6b39ae9..9f8a0a1a99 100644 --- a/docs/client/operations/tables.md +++ b/docs/client/operations/tables.md @@ -111,7 +111,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = create_table.sync( @@ -181,7 +184,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = await create_table.asyncio( @@ -254,7 +260,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = create_table.sync( @@ -327,7 +336,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = await create_table.asyncio( @@ -833,9 +845,11 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : true, "can_collect_statistics" : true, "can_run_checks" : true, @@ -916,9 +930,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1000,9 +1018,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1087,9 +1109,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -1174,9 +1200,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -9475,9 +9505,11 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : true, "can_collect_statistics" : true, "can_run_checks" : true, @@ -9523,9 +9555,11 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : true, "can_collect_statistics" : true, "can_run_checks" : true, @@ -9571,9 +9605,11 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl "deleteStatistics" : true, "deleteCheckResults" : true, "deleteSensorReadouts" : true, - "deleteErrorSamples" : false, - "deleteIncidents" : false + "deleteErrorSamples" : true, + "deleteIncidents" : true, + "deleteChecksConfiguration" : false }, + "advanced_properties" : { }, "can_edit" : true, "can_collect_statistics" : true, "can_run_checks" : true, @@ -9655,9 +9691,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -9708,9 +9748,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -9761,9 +9805,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -9847,9 +9895,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -9900,9 +9952,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -9953,9 +10009,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10042,9 +10102,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10095,9 +10159,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10148,9 +10216,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10237,9 +10309,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10290,9 +10366,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10343,9 +10423,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10465,7 +10549,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = update_table.sync( @@ -10535,7 +10622,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = await update_table.asyncio( @@ -10608,7 +10698,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = update_table.sync( @@ -10681,7 +10774,10 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl ), monitoring_checks=TableMonitoringCheckCategoriesSpec(), partitioned_checks=TablePartitionedCheckCategoriesSpec(), - columns=ColumnSpecMap() + columns=ColumnSpecMap(), + advanced_properties={ + + } ) call_result = await update_table.asyncio( @@ -10743,7 +10839,7 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"connection_name\":\"sample_connection\",\"table_hash\":7188561880498907939,\"target\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"do_not_collect_error_samples_in_profiling\":false,\"always_collect_error_samples_in_monitoring\":false,\"has_any_configured_checks\":true,\"has_any_configured_profiling_checks\":true,\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"partitioned\"},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":false,\"deleteIncidents\":false},\"can_edit\":true,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" + "{\"connection_name\":\"sample_connection\",\"table_hash\":7188561880498907939,\"target\":{\"schema_name\":\"sample_schema\",\"table_name\":\"sample_table\"},\"do_not_collect_error_samples_in_profiling\":false,\"always_collect_error_samples_in_monitoring\":false,\"has_any_configured_checks\":true,\"has_any_configured_profiling_checks\":true,\"run_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true},\"run_profiling_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"profiling\"},\"run_monitoring_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"monitoring\"},\"run_partition_checks_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"enabled\":true,\"checkType\":\"partitioned\"},\"data_clean_job_template\":{\"connection\":\"sample_connection\",\"fullTableName\":\"sample_schema.sample_table\",\"deleteErrors\":true,\"deleteStatistics\":true,\"deleteCheckResults\":true,\"deleteSensorReadouts\":true,\"deleteErrorSamples\":true,\"deleteIncidents\":true,\"deleteChecksConfiguration\":false},\"advanced_properties\":{},\"can_edit\":true,\"can_collect_statistics\":true,\"can_run_checks\":true,\"can_delete_data\":true}" ``` @@ -10810,9 +10906,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10892,9 +10992,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -10977,9 +11081,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, @@ -11062,9 +11170,13 @@ http://localhost:8888/api/connections/{connectionName}/schemas/{schemaName}/tabl delete_statistics=True, delete_check_results=True, delete_sensor_readouts=True, - delete_error_samples=False, - delete_incidents=False + delete_error_samples=True, + delete_incidents=True, + delete_checks_configuration=False ), + advanced_properties={ + + }, can_edit=True, can_collect_statistics=True, can_run_checks=True, diff --git a/docs/client/operations/users.md b/docs/client/operations/users.md index 64742560c3..72c9f88472 100644 --- a/docs/client/operations/users.md +++ b/docs/client/operations/users.md @@ -320,7 +320,7 @@ http://localhost:8888/api/users | Description                     | Data type | Required | |---------------------------------|-----------|-----------------| -|User model|*[DqoCloudUserModel](../models/users.md#dqocloudusermodel)*| | +|User model|*[DqoUserRolesModel](../models/users.md#dqouserrolesmodel)*| | @@ -336,7 +336,7 @@ http://localhost:8888/api/users -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"email\":\"sample_user@mail.com\",\"accountRole\":\"operator\"}" + "{\"email\":\"sample_user@mail.com\",\"accountRole\":\"operator\",\"dataDomainRoles\":{\"(default)\":\"editor\",\"datalake\":\"editor\"}}" ``` @@ -349,16 +349,17 @@ http://localhost:8888/api/users ```python from dqops import client from dqops.client.api.users import create_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel dqops_client = client.Client( 'http://localhost:8888/' ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = create_user.sync( @@ -377,16 +378,17 @@ http://localhost:8888/api/users ```python from dqops import client from dqops.client.api.users import create_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel dqops_client = client.Client( 'http://localhost:8888/' ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = await create_user.asyncio( @@ -405,8 +407,8 @@ http://localhost:8888/api/users ```python from dqops import client from dqops.client.api.users import create_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel token = 's4mp13_4u7h_70k3n' @@ -415,9 +417,10 @@ http://localhost:8888/api/users token=token ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = create_user.sync( @@ -436,8 +439,8 @@ http://localhost:8888/api/users ```python from dqops import client from dqops.client.api.users import create_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel token = 's4mp13_4u7h_70k3n' @@ -446,9 +449,10 @@ http://localhost:8888/api/users token=token ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = await create_user.asyncio( @@ -609,7 +613,7 @@ http://localhost:8888/api/users | Property name | Description                     | Data type | |---------------|---------------------------------|-----------| -|`dqo_cloud_user_model`||*List[[DqoCloudUserModel](../models/users.md#dqocloudusermodel)]*| +|`dqo_user_roles_model`||*List[[DqoUserRolesModel](../models/users.md#dqouserrolesmodel)]*| @@ -637,13 +641,25 @@ http://localhost:8888/api/users ``` [ { "email" : "sample_user@mail.com", - "accountRole" : "operator" + "accountRole" : "operator", + "dataDomainRoles" : { + "(default)" : "editor", + "datalake" : "editor" + } }, { "email" : "sample_user@mail.com", - "accountRole" : "operator" + "accountRole" : "operator", + "dataDomainRoles" : { + "(default)" : "editor", + "datalake" : "editor" + } }, { "email" : "sample_user@mail.com", - "accountRole" : "operator" + "accountRole" : "operator", + "dataDomainRoles" : { + "(default)" : "editor", + "datalake" : "editor" + } } ] ``` @@ -673,17 +689,20 @@ http://localhost:8888/api/users ``` [ - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ] ``` @@ -715,17 +734,20 @@ http://localhost:8888/api/users ``` [ - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ] ``` @@ -760,17 +782,20 @@ http://localhost:8888/api/users ``` [ - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ] ``` @@ -805,17 +830,20 @@ http://localhost:8888/api/users ``` [ - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ), - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ] ``` @@ -841,7 +869,7 @@ http://localhost:8888/api/users/{email} | Property name | Description                     | Data type | |---------------|---------------------------------|-----------| -|[`dqo_cloud_user_model`](../models/users.md#dqocloudusermodel)||*[DqoCloudUserModel](../models/users.md#dqocloudusermodel)*| +|[`dqo_user_roles_model`](../models/users.md#dqouserrolesmodel)||*[DqoUserRolesModel](../models/users.md#dqouserrolesmodel)*| @@ -876,7 +904,11 @@ http://localhost:8888/api/users/{email} ``` { "email" : "sample_user@mail.com", - "accountRole" : "operator" + "accountRole" : "operator", + "dataDomainRoles" : { + "(default)" : "editor", + "datalake" : "editor" + } } ``` @@ -906,9 +938,10 @@ http://localhost:8888/api/users/{email} ??? example "Expand to see the returned result" ``` - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ``` @@ -939,9 +972,10 @@ http://localhost:8888/api/users/{email} ??? example "Expand to see the returned result" ``` - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ``` @@ -975,9 +1009,10 @@ http://localhost:8888/api/users/{email} ??? example "Expand to see the returned result" ``` - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ``` @@ -1011,9 +1046,10 @@ http://localhost:8888/api/users/{email} ??? example "Expand to see the returned result" ``` - DqoCloudUserModel( + DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) ``` @@ -1049,7 +1085,7 @@ http://localhost:8888/api/users/{email} | Description                     | Data type | Required | |---------------------------------|-----------|-----------------| -|User model|*[DqoCloudUserModel](../models/users.md#dqocloudusermodel)*| | +|User model|*[DqoUserRolesModel](../models/users.md#dqouserrolesmodel)*| | @@ -1065,7 +1101,7 @@ http://localhost:8888/api/users/{email} -H "Accept: application/json"^ -H "Content-Type: application/json"^ -d^ - "{\"email\":\"sample_user@mail.com\",\"accountRole\":\"operator\"}" + "{\"email\":\"sample_user@mail.com\",\"accountRole\":\"operator\",\"dataDomainRoles\":{\"(default)\":\"editor\",\"datalake\":\"editor\"}}" ``` @@ -1078,16 +1114,17 @@ http://localhost:8888/api/users/{email} ```python from dqops import client from dqops.client.api.users import update_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel dqops_client = client.Client( 'http://localhost:8888/' ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = update_user.sync( @@ -1107,16 +1144,17 @@ http://localhost:8888/api/users/{email} ```python from dqops import client from dqops.client.api.users import update_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel dqops_client = client.Client( 'http://localhost:8888/' ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = await update_user.asyncio( @@ -1136,8 +1174,8 @@ http://localhost:8888/api/users/{email} ```python from dqops import client from dqops.client.api.users import update_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel token = 's4mp13_4u7h_70k3n' @@ -1146,9 +1184,10 @@ http://localhost:8888/api/users/{email} token=token ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = update_user.sync( @@ -1168,8 +1207,8 @@ http://localhost:8888/api/users/{email} ```python from dqops import client from dqops.client.api.users import update_user - from dqops.client.models import DqoCloudUserModel, \ - DqoUserRole + from dqops.client.models import DqoUserRole, \ + DqoUserRolesModel token = 's4mp13_4u7h_70k3n' @@ -1178,9 +1217,10 @@ http://localhost:8888/api/users/{email} token=token ) - request_body = DqoCloudUserModel( + request_body = DqoUserRolesModel( email='sample_user@mail.com', - account_role=DqoUserRole.OPERATOR + account_role=DqoUserRole.OPERATOR, + data_domain_roles=LinkedHashMap() ) call_result = await update_user.asyncio( diff --git a/docs/command-line-interface/check.md b/docs/command-line-interface/check.md index f018839da7..faccfdac8e 100644 --- a/docs/command-line-interface/check.md +++ b/docs/command-line-interface/check.md @@ -36,7 +36,7 @@ $ dqo [dqo options...] check run [-deh] [-ces] [--daily-partitioning-include-tod s>] [-of=] [-s=] [-t=] [--to-date=] [--to-date-time=] [--to-date-time-offset=] [-ts=] - [-l=]... [-tag=]... + [--where-filter=] [-l=]... [-tag=]... ``` @@ -55,7 +55,7 @@ dqo> check run [-deh] [-ces] [--daily-partitioning-include-today] [-fe] [-fw] [- s>] [-of=] [-s=] [-t=
] [--to-date=] [--to-date-time=] [--to-date-time-offset=] [-ts=] - [-l=]... [-tag=]... + [--where-filter=] [-l=]... [-tag=]... ``` @@ -97,6 +97,7 @@ All parameters supported by the command are listed below. |
`--to-date`
|Analyze the data until the given date (exclusive, the given date and the following dates are not analyzed). The date should be an ISO 8601 date (YYYY-MM-DD). The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date overrides the parameters to disable analyzing today or the current month.| || |
`--to-date-time`
|Analyze the data until the given date and time (exclusive). The date should be an ISO 8601 date (yyyy-MM-dd). The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date and time overrides the parameters to disable analyzing today or the current month.| || |
`--to-date-time-offset`
|Analyze the data until the given date and time with a time zone offset (exclusive). The date and time should be an ISO 8601 date and time followed by a time zone offset (yyyy-MM-dd HH\:mm:ss). For example: 2023-02-20 14:10:00+02. The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date and time overrides the parameters to disable analyzing today or the current month.| || +|
`--where-filter`
|An additional filter which must be a valid SQL predicate (an SQL expression that returns 'true' or 'false') that is added to the WHERE clause of the SQL query that DQOps will run on the data source. The purpose of a custom filter is to analyze only a subset of data, for example, when a new batch of records is loaded, and the data quality checks are evaluated as a data contract. All the records in that batch must tagged with the same value, and the passed predicate to find records from that batch would use the filter in the form: "{alias}.batch_id = 1". The filter can use replacement tokens {alias} to reference the analyzed table.| || diff --git a/docs/command-line-interface/collect.md b/docs/command-line-interface/collect.md index e80e8bd769..1466b005cf 100644 --- a/docs/command-line-interface/collect.md +++ b/docs/command-line-interface/collect.md @@ -38,7 +38,8 @@ $ dqo [dqo options...] collect errorsamples [-deh] [--daily-partitioning-include [-sc=] [-t=
] [--to-date=] [--to-date-time=] [--to-date-time-offset=] - [-ts=] [-l=]... [-tag=]... + [-ts=] [--where-filter=] + [-l=]... [-tag=]... ``` @@ -59,7 +60,8 @@ dqo> collect errorsamples [-deh] [--daily-partitioning-include-today] [-fe] [-fw [-sc=] [-t=
] [--to-date=] [--to-date-time=] [--to-date-time-offset=] - [-ts=] [-l=]... [-tag=]... + [-ts=] [--where-filter=] + [-l=]... [-tag=]... ``` @@ -100,6 +102,7 @@ All parameters supported by the command are listed below. |
`--to-date`
|Analyze the data until the given date (exclusive, the given date and the following dates are not analyzed). The date should be an ISO 8601 date (YYYY-MM-DD). The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date overrides the parameters to disable analyzing today or the current month.| || |
`--to-date-time`
|Analyze the data until the given date and time (exclusive). The date should be an ISO 8601 date (yyyy-MM-dd). The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date and time overrides the parameters to disable analyzing today or the current month.| || |
`--to-date-time-offset`
|Analyze the data until the given date and time with a time zone offset (exclusive). The date and time should be an ISO 8601 date and time followed by a time zone offset (yyyy-MM-dd HH\:mm:ss). For example: 2023-02-20 14:10:00+02. The analyzed table must have the timestamp column properly configured, it is the column that is used for filtering the date and time ranges. Setting the end date and time overrides the parameters to disable analyzing today or the current month.| || +|
`--where-filter`
|An additional filter which must be a valid SQL predicate (an SQL expression that returns 'true' or 'false') that is added to the WHERE clause of the SQL query that DQOps will run on the data source. The purpose of a custom filter is to analyze only a subset of data, for example, when a new batch of records is loaded, and the data quality checks are evaluated as a data contract. All the records in that batch must tagged with the same value, and the passed predicate to find records from that batch would use the filter in the form: "{alias}.batch_id = 1". The filter can use replacement tokens {alias} to reference the analyzed table.| || diff --git a/docs/data-sources/athena.md b/docs/data-sources/athena.md index 05526e267e..c89bd06086 100644 --- a/docs/data-sources/athena.md +++ b/docs/data-sources/athena.md @@ -28,7 +28,7 @@ To navigate to the Athena connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Athena database type. @@ -90,12 +90,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add an Athena connection using DQOps Shell diff --git a/docs/data-sources/aws.md b/docs/data-sources/aws.md index 4dbac7811d..7c3f7f0d8e 100644 --- a/docs/data-sources/aws.md +++ b/docs/data-sources/aws.md @@ -132,7 +132,7 @@ DQOps uses the DuckDB connector to work with AWS S3 buckets. To navigate to the 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the DuckDB connection. @@ -205,12 +205,24 @@ Now we can import files. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-csv.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-csv.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Details of new connection - all parameters description diff --git a/docs/data-sources/azure.md b/docs/data-sources/azure.md index 75dcf80c60..dfd7c204a1 100644 --- a/docs/data-sources/azure.md +++ b/docs/data-sources/azure.md @@ -157,7 +157,7 @@ DQOps uses the DuckDB connector to work with Azure Blob Storage. To navigate to 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the DuckDB connection. @@ -233,12 +233,24 @@ Now we can import files. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-csv.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-csv.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Details of new connection - all parameters description diff --git a/docs/data-sources/bigquery.md b/docs/data-sources/bigquery.md index 8fb670c974..5a1b84fa7e 100644 --- a/docs/data-sources/bigquery.md +++ b/docs/data-sources/bigquery.md @@ -25,7 +25,7 @@ To navigate to the BigQuery connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select BiqQuery database type. @@ -70,12 +70,24 @@ Click the **Save** connection button when the test is successful otherwise, you ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png) -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, table -availability and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. By clicking -on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, or modify -the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png) +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a BigQuery connection using DQOps Shell diff --git a/docs/data-sources/csv.md b/docs/data-sources/csv.md index d6599fc4d4..65c1fd1b79 100644 --- a/docs/data-sources/csv.md +++ b/docs/data-sources/csv.md @@ -25,11 +25,11 @@ To navigate to the CSV connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the CSV file connection option. - ![Selecting CSV database type](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection-csv.png){ loading=lazy; width="1200px" } + ![Selecting CSV database type](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection-csv2.png){ loading=lazy; width="1200px" } ### **Fill in the connection settings** @@ -167,12 +167,24 @@ Now we can import CSV files. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-csv.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-csv.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ### Register single file as table diff --git a/docs/data-sources/databricks.md b/docs/data-sources/databricks.md index 64c736ea5b..ca3cd2fd18 100644 --- a/docs/data-sources/databricks.md +++ b/docs/data-sources/databricks.md @@ -23,7 +23,7 @@ To navigate to the Databricks connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Databricks database type. @@ -83,12 +83,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a Databricks connection using DQOps Shell diff --git a/docs/data-sources/deltalake.md b/docs/data-sources/deltalake.md index 976c5e9655..88b93f98a4 100644 --- a/docs/data-sources/deltalake.md +++ b/docs/data-sources/deltalake.md @@ -24,7 +24,7 @@ To navigate to the Delta Lake connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the Delta Lake table connection option. @@ -116,12 +116,24 @@ Now we can import Delta Lake table. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-deltalake.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-deltalake.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a Delta Lake connection using DQOps Shell diff --git a/docs/data-sources/duckdb.md b/docs/data-sources/duckdb.md index 099d8479ee..47843ee7d1 100644 --- a/docs/data-sources/duckdb.md +++ b/docs/data-sources/duckdb.md @@ -25,7 +25,7 @@ To navigate to the DuckDB connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the DuckDB file connection option. diff --git a/docs/data-sources/gcp.md b/docs/data-sources/gcp.md index 7e38b21c9d..91a1bcd576 100644 --- a/docs/data-sources/gcp.md +++ b/docs/data-sources/gcp.md @@ -34,7 +34,7 @@ DQOps uses the DuckDB connector to work with Google Cloud Storage buckets. To na 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the DuckDB connection. @@ -109,13 +109,24 @@ Now we can import files. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-csv.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-csv.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Details of new connection - all parameters description diff --git a/docs/data-sources/iceberg.md b/docs/data-sources/iceberg.md index a321a1f553..7396b9f4f4 100644 --- a/docs/data-sources/iceberg.md +++ b/docs/data-sources/iceberg.md @@ -24,7 +24,7 @@ To navigate to the Iceberg connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the Iceberg table connection option. @@ -132,12 +132,24 @@ Now we can import Iceberg table. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-iceberg.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-iceberg.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add an Iceberg connection using DQOps Shell diff --git a/docs/data-sources/json.md b/docs/data-sources/json.md index 518596ce3c..60fdcefd94 100644 --- a/docs/data-sources/json.md +++ b/docs/data-sources/json.md @@ -25,7 +25,7 @@ To navigate to the JSON connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the JSON file connection option. @@ -165,12 +165,24 @@ Now we can import JSON files. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-json.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-json.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ### Register single file as table diff --git a/docs/data-sources/mysql.md b/docs/data-sources/mysql.md index 94b9228d64..b5d7634f05 100644 --- a/docs/data-sources/mysql.md +++ b/docs/data-sources/mysql.md @@ -26,7 +26,7 @@ To navigate to the MySQL connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select MySQL database type. @@ -86,13 +86,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a MySQL connection using DQOps Shell diff --git a/docs/data-sources/oracle.md b/docs/data-sources/oracle.md index 56bded8c69..e5e92fc457 100644 --- a/docs/data-sources/oracle.md +++ b/docs/data-sources/oracle.md @@ -17,7 +17,7 @@ To navigate to the Oracle connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Oracle database type. @@ -72,14 +72,24 @@ Click the **Save** connection button when the test is successful otherwise, you ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +!!! info "Automatically activated checks" + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add an Oracle connection using DQOps Shell diff --git a/docs/data-sources/parquet.md b/docs/data-sources/parquet.md index 731ffdfd9c..112e6f4250 100644 --- a/docs/data-sources/parquet.md +++ b/docs/data-sources/parquet.md @@ -25,7 +25,7 @@ To navigate to the Parquet connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select the Parquet file connection option. @@ -154,13 +154,24 @@ Now we can import Parquet files. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-parquet.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/duckdb/importing-tables-advisor-parquet.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ### Register single file as table diff --git a/docs/data-sources/postgresql.md b/docs/data-sources/postgresql.md index 7a4936179e..ffe49d2fd0 100644 --- a/docs/data-sources/postgresql.md +++ b/docs/data-sources/postgresql.md @@ -25,7 +25,7 @@ To navigate to the PostgreSQL connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select PostgreSQL database type. @@ -79,12 +79,24 @@ Click the **Save** connection button when the test is successful otherwise, you ![AImporting tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.jpg) -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a PostgreSQL connection using DQOps Shell diff --git a/docs/data-sources/presto.md b/docs/data-sources/presto.md index d6975a6ace..de8398a016 100644 --- a/docs/data-sources/presto.md +++ b/docs/data-sources/presto.md @@ -16,7 +16,7 @@ To navigate to the Presto connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Presto database type. @@ -75,13 +75,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a Presto connection using DQOps Shell diff --git a/docs/data-sources/redshift.md b/docs/data-sources/redshift.md index bd1c7bc28b..18df8e1127 100644 --- a/docs/data-sources/redshift.md +++ b/docs/data-sources/redshift.md @@ -25,7 +25,7 @@ To navigate to the Redshift connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Redshift database type. @@ -85,14 +85,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks -or modify the schedule for newly imported tables. +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +!!! info "Automatically activated checks" + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a Redshift connection using DQOps Shell diff --git a/docs/data-sources/single-store.md b/docs/data-sources/single-store.md index ae3b94c3c0..6e140f44a8 100644 --- a/docs/data-sources/single-store.md +++ b/docs/data-sources/single-store.md @@ -20,7 +20,7 @@ To navigate to the SingleStoreDB connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select SingleStoreDB database type. @@ -79,14 +79,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png) +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png) +!!! info "Automatically activated checks" + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a SingleStoreDB connection using DQOps Shell diff --git a/docs/data-sources/snowflake.md b/docs/data-sources/snowflake.md index 7515090707..6132c8cbfb 100644 --- a/docs/data-sources/snowflake.md +++ b/docs/data-sources/snowflake.md @@ -24,7 +24,7 @@ To navigate to the Snowflake connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Snowflake database type. @@ -80,14 +80,24 @@ Click the **Save** connection button when the test is successful otherwise, you ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +!!! info "Automatically activated checks" + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a Snowflake connection using DQOps Shell diff --git a/docs/data-sources/spark.md b/docs/data-sources/spark.md index 771a100ef4..31564f5d54 100644 --- a/docs/data-sources/spark.md +++ b/docs/data-sources/spark.md @@ -20,7 +20,7 @@ To navigate to the Spark connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Spark database type. @@ -78,13 +78,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a Spark connection using DQOps Shell diff --git a/docs/data-sources/sql-server.md b/docs/data-sources/sql-server.md index ed9058e995..1aeceebe16 100644 --- a/docs/data-sources/sql-server.md +++ b/docs/data-sources/sql-server.md @@ -24,7 +24,7 @@ To navigate to the SQL Server connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.jpg) + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select SQL Server database type. @@ -83,14 +83,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +!!! info "Automatically activated checks" + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add an SQL Server connection using DQOps Shell diff --git a/docs/data-sources/trino.md b/docs/data-sources/trino.md index bf4af9f136..c2192ed9fd 100644 --- a/docs/data-sources/trino.md +++ b/docs/data-sources/trino.md @@ -16,7 +16,7 @@ To navigate to the Trino connection settings: 1. Go to the Data Sources section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png){ loading=lazy; width="1200px" } + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select Trino database type. @@ -76,13 +76,24 @@ Now we can import schemas and tables. ![Importing tables](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, such as row count, -table availability, and checks detecting schema changes. These checks are scheduled to run daily at 12:00 p.m. -By clicking on the Advisor at the top of the page, you can quickly collect basic statistics, run profiling checks, -or modify the schedule for newly imported tables. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/adding-connections/importing-tables-advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Add a Trino connection using DQOps Shell diff --git a/docs/dqo-concepts/architecture/data-quality-check-execution-flow.md b/docs/dqo-concepts/architecture/data-quality-check-execution-flow.md index b2d5b2bd2a..1679e5cb88 100644 --- a/docs/dqo-concepts/architecture/data-quality-check-execution-flow.md +++ b/docs/dqo-concepts/architecture/data-quality-check-execution-flow.md @@ -86,7 +86,7 @@ The following steps are performed by the DQOps engine to run a data quality chec First, the rule for the *fatal* severity level is called. If the rule fails, the check result is a failure and is scored at a *fatal* severity level and no further rules (*error* or *warning* severity level rules) are evaluated. - If the *fatal* severity rule is not activated or has passed (accepting the *sensor readout* as a valid result), DQOps continues + If the *fatal* severity rule is not activated or has passed (accepting the *sensor readout* as a correct result), DQOps continues the rule evaluation by evaluating the *error* severity rule. If the *error* severity rule also passes, the *warning* severity rule is evaluated. Failed data quality checks are called *data quality issues* or just `issues` on the [data quality dashboards](../types-of-data-quality-dashboards.md). diff --git a/docs/dqo-concepts/configuring-data-quality-checks-and-rules.md b/docs/dqo-concepts/configuring-data-quality-checks-and-rules.md index 34c68cd222..be04f13a67 100644 --- a/docs/dqo-concepts/configuring-data-quality-checks-and-rules.md +++ b/docs/dqo-concepts/configuring-data-quality-checks-and-rules.md @@ -420,6 +420,10 @@ adding also standalone values included in the list. The definition of custom data dictionaries is described in the [data dictionaries](dqops-user-home-folder.md#data-dictionaries) section of the *DQOps user home* folder concept. +You can define dictionaries in the **Configuration** section of the user interface. + +![Defining dictionaries in the user interface](https://dqops.com/docs/images/concepts/configuring-data-quality-checks-and-rules/defining-dictionaries-in-the-user-interface1.png){ loading=lazy; width="1200px" } + ## Configuring column-level checks The list of columns is stored in the `spec.columns` node in the *.dqotable.yaml* file. The [configuration of the column metadata](configuring-table-metadata.md#configuring-columns) is described @@ -817,6 +821,7 @@ To learn more about [configuring multiple data quality checks](../working-with-d ## What's next +- Data quality checks do not need to be configured manually. Learn how the [data quality rule mining](data-quality-rule-mining.md) engine can automatically propose and configure data quality checks to detect the most common data quality issues. - You haven't installed DQOps yet? Check the detailed guide on how to [install DQOps using pip](../dqops-installation/install-dqops-using-pip.md) or [run DQOps as a Docker container](../dqops-installation/run-dqops-as-docker-container.md). - DQOps has multiple built-in data quality dashboards for displaying data quality KPI. [Learn more about different types of dashboards](types-of-data-quality-dashboards.md). - DQOps allows you to keep track of the issues that arise during data quality monitoring and send alert notifications directly to Slack. Learn more about [incidents](../working-with-dqo/managing-data-quality-incidents-with-dqops.md) and [notifications](../integrations/webhooks/index.md). diff --git a/docs/dqo-concepts/data-quality-dimensions.md b/docs/dqo-concepts/data-quality-dimensions.md index 2f011e4d73..c94e29ce00 100644 --- a/docs/dqo-concepts/data-quality-dimensions.md +++ b/docs/dqo-concepts/data-quality-dimensions.md @@ -468,3 +468,13 @@ The following table lists data quality checks that detect validity issues on col ## What's more - Review the [types of data quality dashboards](types-of-data-quality-dashboards.md) that are provided with DQOps. - Learn how DQOps calculates [data quality KPIs per data quality dimension](definition-of-data-quality-kpis.md#monthly-with-data-quality-dimensions). + +### Table uniqueness checks +The following table lists data quality checks that detect uniqueness issues on tables. + +| Data quality check name | Friendly name | Check category | Description | Standard check | +|-------------------------|---------------|----------------|-------------|----------------| +|[*duplicate_record_count*](../checks/table/uniqueness/duplicate-record-count.md)|Maximum count of duplicate records|[uniqueness](../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|This check counts duplicate records values. It raises a data quality issue when the number of duplicates is above a minimum accepted value. The default configuration detects duplicate rows by enforcing that the *min_count* of duplicates is zero.|:material-check-bold:| +|[*duplicate_record_percent*](../checks/table/uniqueness/duplicate-record-percent.md)|Maximum percentage of duplicate records|[uniqueness](../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md)|This check measures the percentage of duplicate records values. It raises a data quality issue when the percentage of duplicates is above a minimum accepted value. The default threshold is 0% duplicate values.|:material-check-bold:| + +## End of list, remove this entry and move the content up diff --git a/docs/dqo-concepts/data-quality-error-sampling.md b/docs/dqo-concepts/data-quality-error-sampling.md index 2ef6feeb9d..39b43b3aab 100644 --- a/docs/dqo-concepts/data-quality-error-sampling.md +++ b/docs/dqo-concepts/data-quality-error-sampling.md @@ -23,22 +23,38 @@ To collect error samples, just run the check manually: 3. Activate the check of interest, set the thresholds or leave the default options. 4. Click on the check Run icon. - To view the error samples, click on the **Results** icon, and select the tab **Error sampling**. For column-level checks that do not have error samples, the tab will be inactive. -![View error samples](https://dqops.com/docs/images/concepts/data-quality-error-sampling/view-error-samples.png){ loading=lazy; width="1200px" } +![View error samples](https://dqops.com/docs/images/concepts/data-quality-error-sampling/view-error-samples2.png){ loading=lazy; width="1200px" } The table with error samples has the following columns: - **Sample index:** The index of the collected sample. - **Collected at:** Column for the time when the error samples were captured. All error samples results started as part of the same error sampling session will share the same time. -- **Result:** The sample value. - **Result data type:** The sample's result data type. +- **Result:** The sample value. - **ID Column 1:** The value of the ID column from the original data row that corresponds to the error sample. This column helps trace the error back to its source record. You can set up to 9 ID Columns. -- **Data grouping:** The data grouping name - **Id:** The error sample result id (primary key). This value identifies a single row. +## Download error samples as a CSV file +To aid in root cause analysis and data cleanup, DQOps enables you to download error samples as a CSV file. +These samples provide a representative selection of data points that failed to meet the specified data quality criteria. + +To download error samples: + +1. Navigate to the data quality check with available results. +2. Click on the **Results** icon to open the results. +3. Choose the **Error sampling** tag. +4. Click the **Download as CSV** icon button. +5. Your browser will start the download. + +The CSV file you download will include up to 100 sample records that did not meet the defined data quality thresholds for the specific check. + +![Download error samples as a CSV file](https://dqops.com/docs/images/concepts/data-quality-error-sampling/download-error-samples-as-csv-file.png){ loading=lazy; width="1200px" } + +By analyzing error samples, you can gain valuable insights into the nature of data quality issues, identify patterns, and take appropriate corrective actions to improve your data. + ## Configuring an ID column To be able to collect error samples you need to define an ID column to uniquely identify error samples. Many database systems inherently include ID columns (such as auto-incrementing fields in SQL Server or MySQL). diff --git a/docs/dqo-concepts/data-quality-process.md b/docs/dqo-concepts/data-quality-process.md new file mode 100644 index 0000000000..cf782cd3e7 --- /dev/null +++ b/docs/dqo-concepts/data-quality-process.md @@ -0,0 +1,281 @@ +--- +title: Data quality process +--- +# Data quality process +Read this guide to learn the whole DQOps data quality process, from data profiling and data quality assessment, to data observability and incident management. + +## Data quality process overview +Ensuring data quality requires a process that has two main stages. +The first stage is the data quality assessment, which focuses on understanding the dataset's structure and finding data quality issues. +It is followed by an optional data cleansing step to fix identified issues. + +The second stage focuses on data quality monitoring to detect data quality issues in the future. +This stage is also called data observability and depends on running data quality checks and anomaly detection triggered by a scheduler. + + +### Problem +The purpose of data quality is to find data quality issues that make the datasets unusable for their purpose, +such as deriving insights on dashboards or using them as training data in an AI project. + +### Goal +Data teams that want to ensure data quality in their data platforms have three goals to achieve. + +* Measure the health (data quality status) of each table. + +* Fix the most severe data quality issues, ensuring that critical data assets are usable for their purpose. + +* Detect new data quality issues automatically in the future. + +### Solution +Achieving these goals requires a data quality platform that supports data observability to detect future issues. + +* The data quality platform should help users perform data quality assessments and measure each table's baseline data quality status. + +* The platform should help the data teams find and fix invalid records. + +* Data observability is applied to detect anomalies and revalidate data quality checks at regular intervals, such as daily. + + +## Data quality lifecycle +``` mermaid +graph LR + DATA_QUALITY_ASSESSMENT[Data quality
assessment]:::assessment --> |Identify issues
and required
data quality checks| DATA_QUALITY_REVIEW[Review data
quality issues
with data owners]:::review; + DATA_QUALITY_REVIEW --> |Valid issues
confirmed| DATA_CLEANSING[Data cleansing
 to fix data
quality issues]:::cleansing; + DATA_CLEANSING --> |Activate
data observablity| DATA_OBSERVABILITY[Data quality
monitoring]:::monitoring; + + classDef assessment fill:#57ff6a; + classDef review fill:#57e3ff; + classDef cleansing fill:#5795ff; + classDef monitoring fill:#fccf86; +``` + +The data quality lifecycle is a holistic process of ensuring data quality. +Its purpose is to allow data teams to ensure data quality in a repeatable way and preserve high data quality in a long-term perspective. + +It covers all steps, from learning about the structure of a table, selecting and evaluating data quality checks, reviewing detected data quality issues, +fixing these issues by data cleansing, and enabling long-term data quality monitoring to detect when the issue reappears instantly. + +The data quality lifecycle stages are described in the following sections, which also show how DQOps supports this process. + +### Data quality assessment +``` mermaid +graph LR + BASIC_PROFILING[Basic statistics
collection]:::step --> |Propose data
quality checks| RULE_MINING[Data quality
rule mining]:::step; + RULE_MINING --> |Run proposed
checks| ASSESSMENT_RESULTS_REVIEW[Review initial
data quality KPI]:::step; + ASSESSMENT_RESULTS_REVIEW --> |Correct data
quality rules| DISABLE_FALSE_CHECKS[Disable false-positive
checks and tweak rules]:::step; + DISABLE_FALSE_CHECKS --> |Share data quality
status report| CONSULT_DQ_ISSUES[Review data quality
issues with the data owner]:::step; + + classDef step fill:#57ff6a; +``` + +The purpose of performing a data quality assessment of a table is to understand its structure and data distribution +and find data quality issues. Data teams that will be using a dataset to ingest it into their data platform or data consumers, +such as data analysts or data scientists, can perform the data quality assessment to verify if the dataset is usable +for their use cases and how much data cleansing and transformation is required to make the data usable. + +Data quality assessment is a routine activity for data governance and quality teams. +DQOps makes this process simpler and possible from a local computer without setting up any complex data quality platforms. +The data quality assessment of tables is performed in the data profiling module in DQOps. + + +!!! tip "DQOps data quality rule mining is a must-have tool for every software company dealing with data and AI projects" + + If you are starting an AI or Data & Analytics project for a customer, and the data quality is a concern, it should be measured in the project's first stage. + As soon as you have access to the customer's database or data lake, you should start DQOps locally on your laptop in a new empty folder, + import the metadata of the tables, and let the rule mining engine find the most prominent data quality issues of all the tables that you will be using in the project. + + If you see any data quality issue that can pose a severe risk to the delivery of an AI or analytics project, + use the table quality status screen to show the customer the data health status and request an additional budget + for performing data cleansing before the poor data quality will affect the project's delivery timeline and goals. + + +DQOps simplifies the data quality assessment process by combining two DQOps features. +The first step of data quality assessment is basic data profiling. +DQOps collects statistical information on each column and collects sample values from an analyzed table. + +The [basic data statistics](../working-with-dqo/collecting-basic-data-statistics.md) screen showing selected statistics is shown below. + +![Data profiling statistics screen in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-assessment-statistics-min.png){ loading=lazy; width="1200px" } + +The next step of data quality assessment is validating the data with data quality checks to ensure that +it meets its data quality requirements and is not affected by data quality issues. +Without a capable data quality platform, data professionals must run tens or hundreds of manually +written SQL queries to verify all aspects of data that could be potential data quality issues, such as too many null values. + +DQOps users, such as data engineers, data stewards, or data quality teams, +can speed up this process by using the DQOps rule mining engine to propose the configuration of data quality checks +to detect the most common data quality issues. DQOps uses statistics and data samples to propose these checks and their rule thresholds (parameters). + +The following screenshot shows the proposition of some data quality checks that should detect issues if less than 2% of records are invalid. + +![Data quality rule mining screen in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-assessment-rule-mining-min.png){ loading=lazy; width="1200px" } + + +### Review data quality issues +``` mermaid +graph LR + EXPORT_DATA_QUALITY_REPOT[Export data
quality report]:::step --> |Send issue report
for review| REVIEW_ISSUES[Review and confirm
data quality issues
by data owner]:::step; + REVIEW_ISSUES --> |Receive confirmation
of valid issues| PLAN_DATA_CLEANSING[Plan data cleansing
and tune data quality rules]:::step; + + classDef step fill:#57e3ff; +``` + +After performing the data quality assessment, the user will identify data quality issues that should be fixed. The data quality issues will fall into two categories: + +* Invalid data in the data sources, which requires the engagement of the data source's owner to fix. + For example, the data must be fixed in a line-of-business application (i.e., CRM, ERP). + +* Data transformation issues that can be corrected in the data pipeline. + +The tasks for fixing confirmed data quality issues should be planned for the data cleansing tasks. +False-positive issues will require disabling incorrectly configured data quality checks. + +DQOps shows the [data quality status](dqops-user-interface-overview.md#table-quality-status) for each table in the form of a matrix. +Each row shows the data quality health status for one table's row. +The columns on this view group checks into [categories of data quality checks](../categories-of-data-quality-checks/index.md), +but an alternative view to show [data quality dimensions](data-quality-dimensions.md) is also supported. + +The data quality checks are configured to raise data quality issues at various [severity levels](definition-of-data-quality-checks/index.md#issue-severity-levels). +Warnings for minor problems, errors for problems that require fixing within a reasonable time, +and fatal issues that require stopping data pipelines and applying data cleansing immediately. +The colors of cells in the matrix show the severity status of the most severe data quality issue for each column and category of checks. + +![Table data quality status in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-table-quality-status-min.png){ loading=lazy; width="1200px" } + +DQOps also supports editing the configuration of data quality checks +and reviewing the most recent data quality check status on the [data quality check editor](dqops-user-interface-overview.md#check-editor) screen shown below. + +![Data quality check editor in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-assessment-check-editor-min.png){ loading=lazy; width="1200px" } + + +### Data cleansing +``` mermaid +graph LR + REVIEW_ERROR_SAMPLES[Review
error samples]:::step --> |Send errors
to data owner| FIX_IN_DATA_SOURCE[Fix data quality issues
in the data source]:::step; + FIX_IN_DATA_SOURCE --> |Issues fixed
in the data source| FIX_DATA_PIPELINES[Fix transformations
in data pipelines]:::step; + FIX_DATA_PIPELINES --> |Data
fixed| VALIDATE_WITH_CHECKS[Validate fixes
with data quality checks]:::step; + + classDef step fill:#5795ff; +``` + +The responsibility of fixing data cleansing tasks lies with the business platform owners, who are equipped to address issues in line-of-business applications. +The issues that can be fixed in the data pipelines should be fixed by updating the data transformation logic in the data pipelines. Data engineering teams perform this activity. + +After the data is fixed, the data profiling checks previously configured in DQOps should be rerun to confirm that the data quality issues were fixed. + +DQOps assists in data cleansing by [capturing data quality error samples](data-quality-error-sampling.md) from a subset of invalid records. +The following screen shows how DQOps detected column values not in the list of accepted categorical values. + +![Data quality errors captured by DQOps error sampling](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-error-sampling-min.png){ loading=lazy; width="1200px" } + + +### Data quality monitoring +``` mermaid +graph LR + MINE_MONITORING_CHECKS[Configure
monitoring checks
using rule miner]:::step --> |Full table-scan
checks configured| CONFIGURE_CRON_SCHEDULES[Configure CRON
schedules]:::step; + MINE_PARTITION_CHECKS[Configure
partition checks
using rule miner]:::step --> |Partition checks
configured| CONFIGURE_CRON_SCHEDULES; + CONFIGURE_CRON_SCHEDULES --> |Issues detected
by data
observability| ISSUE_DETECTED[Issue detected
and data quality
incident created]:::step; + ISSUE_DETECTED --> |Send
notification| REVIEW_AND_CONFIRM_ISSUE[Review and
acknowledge
new incidents]:::step; + REVIEW_AND_CONFIRM_ISSUE --> |Incident
acknowledged| FIX_DATA[Fix data
or data pipelines]:::step; + FIX_DATA --> |Issues
fixed| REVALIDATE_DATA_WITH_CHECKS[Revalidate data
with data
quality checks]:::step; + + classDef step fill:#fccf86; +``` + +As soon as a valid set of data profiling checks is selected and false-positive data quality checks are disabled, the user is ready to activate continuous data quality monitoring. + +DQOps supports two types of continuous data quality monitoring checks: + +* [Monitoring checks](definition-of-data-quality-checks/data-observability-monitoring-checks.md) perform full-table scans in daily or monthly periods. +* [Partition checks](definition-of-data-quality-checks/partition-checks.md) evaluate the quality of each daily or monthly partition for append-only or huge tables. + +These two types of checks are configured in the *Monitoring* and *Partition* sections of the [DQOps user interface](dqops-user-interface-overview.md). +DQOps will evaluate the configured data quality checks and raise [data quality incidents](grouping-data-quality-issues-to-incidents.md#incident-management) when data quality issues are detected. +Data operations or support teams will be notified by email or by [incident notifications](grouping-data-quality-issues-to-incidents.md#incident-notifications). + +DQOps captures data quality measures and data quality results for each day, +allowing regulatory compliance and measuring the reliability score ([data quality KPI](definition-of-data-quality-kpis.md)) for each table. + +The following screen shows a data quality check editor screen for a table monitoring check that validates +the [minimum row count](../categories-of-data-quality-checks/how-to-detect-data-volume-issues-and-changes.md#minimum-row-count). +The "check results" tab lists historical data quality results, showing the minimum accepted row count +(*440281* records proposed by the rule mining engine) and the actual row count observed for each day. + +![Data quality monitoring screen in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-monitoring-checks-min.png){ loading=lazy; width="1200px" } + +DQOps measures the health of each table by calculating a data quality KPI score for each table, column, and data quality dimension. +The [data quality KPI formula](definition-of-data-quality-kpis.md#data-quality-score-formula) used by DQOps +is derived as the percentage of successful data quality checks out of all executed data quality checks during the current reporting period, which is the current month. + +The data quality KPI below 100% is a piece of information for the data consumer that says that the table +is currently affected by data quality issues or it was affected by issues recently. +The user should not fully trust that the dataset will be healthy tomorrow. + +The color of each cell identifies the highest [severity level](definition-of-data-quality-checks/index.md#issue-severity-levels) +(warning, error, fatal error) for an active data quality issue detected during the most recent data quality check evaluation. + +![Data quality KPIs for tables screenshot in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-table-kpis-list-min.png){ loading=lazy; width="1200px" } + +The DQOps data quality process depends on two stages. In the first stage, users are performing +the data quality assessment using data quality checks available in the **Profiling** section of the application. +The user should review the data quality status of the table after running profiling checks on it. +The false-positive data quality checks should be disabled, or their data quality rule thresholds should be tweaked to match requirements and the actual data distribution. + +Once the user has configured all [profiling checks](definition-of-data-quality-checks/data-profiling-checks.md), +DQOps requires a second stage to copy **approved** data quality checks from the profiling section to the **Monitoring checks** or **Partition checks** sections. +This process is assisted by using the rule mining engine to copy profiling checks and their parameters. + +The data observability engine will run the checks copied to the monitoring or partition check sections, +and the data quality results will be checkpointed in the local [data quality data lake](architecture/dqops-architecture.md#data-quality-data-lake). +DQOps stores the [results of all data quality checks](data-storage-of-data-quality-results.md) +locally in Parquet files structured to be compatible with any data lake. + +![Configuring data quality monitoring checks in DQOps using rule miner](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-rule-mining-monitoring-checks-min.png){ loading=lazy; width="1200px" } + +An alternative method of configuring data quality checks is using only [data quality policies](data-observability.md) managed by the data observability engine. +The policies are named configurations of data quality checks that are activated on every table or column that matches a filter. +For example, the filter can target all columns named as `*_id` in tables named `fact_*`. +The data quality policy configuration will activate not null checks for all these tables without using the rule engine or the data quality check editor. + +An example of a data quality check named "*Abnormal change in row count since the last known value*" +shown on one of the previous screenshots of the check editor shows such a check. +Its on/off switch control uses a light green color instead of regular green for checks configured manually or using the [rule engine](data-quality-rule-mining.md). + + +## Data quality incident management +The final step in ensuring a good data quality level is setting up the operational processes within the data team. +DQOps will run monitoring and partition data quality checks using its [built-in CRON scheduler](../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md). +However, [triggering data quality checks from data pipelines](../integrations/airflow/index.md) +using the [DQOps client library](../client/index.md) is also supported. + +DQOps groups similar data quality issues into [data quality incidents](grouping-data-quality-issues-to-incidents.md) +to avoid [sending notifications to data teams](grouping-data-quality-issues-to-incidents.md#incident-notifications) too often. +The data quality incidents group similar issues for each table, and all similar data quality issues detected until the incident is resolved are attached to an open incident. + +The following screenshot shows the incident management screen, showing a list of recent data quality incidents. + +![List of data quality incidents screen in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-incidents-list-min.png){ loading=lazy; width="1200px" } + + +The details of the incident are shown after clicking a single incident. DQOps supports performing various actions on the incident, such as: + +* [Acknowledging the incident](grouping-data-quality-issues-to-incidents.md#incident-workflow) activates a workflow to notify a data engineering team that they should fix it. + +* The user can also [disable data quality checks](grouping-data-quality-issues-to-incidents.md#disable-check-for-the-incident) if it was generated by misconfigured checks. + +* DQOps also supports tweaking the data quality rule thresholds to + [decrease the frequency of similar incidents](grouping-data-quality-issues-to-incidents.md#reconfigure-check-for-the-incident) being raised. + It is primarily essential for decreasing the sensitivity of [anomaly detection](../categories-of-data-quality-checks/how-to-detect-anomaly-data-quality-issues.md). + +![Data quality incident screen in DQOps](https://dqops.com/docs/images/concepts/data-quality-process/dq-process-incidents-detail-min.png){ loading=lazy; width="1200px" } + +The users can also [configure incident notifications](grouping-data-quality-issues-to-incidents.md#incident-notifications) +and route notifications to correct data teams based on filters, such as the connection or table name. + + +## What's next + +- Learn how to import [metadata from a data source](configuring-data-sources.md) into DQOps +- Learn how to import [configure data quality checks](configuring-data-quality-checks-and-rules.md) into DQOps +- Understand the [concept of DQOps user interface](dqops-user-interface-overview.md) +- Learn how to use the DQOps [data quality rule miner](data-quality-rule-mining.md) to automate the data quality process diff --git a/docs/dqo-concepts/data-quality-rule-mining.md b/docs/dqo-concepts/data-quality-rule-mining.md new file mode 100644 index 0000000000..66b3879094 --- /dev/null +++ b/docs/dqo-concepts/data-quality-rule-mining.md @@ -0,0 +1,342 @@ +--- +title: Data quality rule mining +--- +# Data quality rule mining +Read this guide to learn how DQOps automatically configures (mines) the rule thresholds in data quality checks to find the most common data quality issues. + +## Definition +Data quality rule mining automatically configures data quality checks, focusing on finding the most probable data quality issues. + +### What is data quality rule mining +Setting up data quality rules by hand and choosing the right threshold can take a lot of time. +This is especially true because most of the time, data quality checks will show that everything is okay. +The real reason we set up these checks is to find those few data quality problems that are hard to see just by looking at the data. + +DQOps makes this process automatic with its automated data quality rule mining engine. +This engine analyzes both [statistical summaries](../working-with-dqo/collecting-basic-data-statistics.md) of the monitored table and sample values from the columns. +You can specify a desired percentage of rows that can contain invalid records. +DQOps will then examine the sample data and propose configurations for data quality checks that can detect the most common types of issues. +These might include a few duplicate rows in a large table, or some emails that don't match a well-known email format. + +Besides configuring checks to find existing problems, DQOps also proposes checks that will pass for the current data. +These checks serve as a baseline. If the data drifts over time and its distribution changes, these checks will fail, +indicating that the data no longer resembles its structure at the time of profiling. + +### Data profiling stages +DQOps uses a two-step process to check your data, which is more thorough than the usual way described in books. +Typically, data profiling just gathers [basic data statistics](../working-with-dqo/collecting-basic-data-statistics.md) +and some sample values from a table and its columns. The tool then shows you things like how many missing or duplicate values there are, +and you have to decide if that's okay and set up the data quality checks yourself. + +DQOps collects basic statistics automatically for every table you register in DQOps. +But it also has a second step that's more advanced. It actually tests your data with different checks to find the most common data quality issues. +You can do this step a few ways: by hand using the [data quality check editor](dqops-user-interface-overview.md#check-editor), +semi-automatically by enabling data quality checks for all matching tables and columns using [data quality policies](data-observability.md), +or you can let DQOps suggest the configuration of [data quality checks](definition-of-data-quality-checks/index.md) for you and then review the problems it finds in your data. + +## Information used for proposing checks +DQOps uses all information collected during basic profiling to propose a reasonable configuration of data quality checks. The data sources are described below. + + +### Basic data statistics +The data quality rule miner uses the following data captured as basic statistics to configure data quality rule thresholds. + +* [Table volume](../categories-of-data-quality-checks/how-to-detect-data-volume-issues-and-changes.md) to configure the minimum accepted row count. + +* [Column counts](../categories-of-data-quality-checks/how-to-detect-table-schema-changes.md) to configure the expected column count and detect when it changes. + +* The [range of values in numeric columns](../categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-numeric-fields.md), + especially the minimum, maximum, and mean (average) values. + DQOps will set up data quality checks to detect when values out of the range appear in the table. + +* The range of date and timestamp columns. DQOps will detect obviously invalid dates, such as *1970-01-01* or *2099-12-31*, + which are often used as placeholder values for null values. The most recent date found in the column used to configure [data timeliness checks](../categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md) + (the timestamp of an event or transaction) will be used to configure a maximum delay in [data freshness checks](../categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md#data-freshness). + +* The [text length](../categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-text-fields.md) + statistics of text columns, such as the minimum, maximum, and mean length of text columns. + Similarly to numeric columns, DQOps will configure checks that will detect text values that are too short or too long in the future. + +* The minimum and maximum [word count of text columns](../checks/column/text/min-word-count.md), + which can provide a helpful way to analyze the quality of data used for Generative AI use cases. + +* [Detected data types](../categories-of-data-quality-checks/how-to-detect-data-type-changes.md) in text columns containing, + if the column contains only boolean, numeric, or date values. The check configured by the DQOps rule miner can be used to analyze raw data in landing zones. + +* [Data completeness](../categories-of-data-quality-checks/how-to-detect-empty-or-incomplete-columns-with-nulls.md) statistics, + such as the number of null values in a column. When a column contains just a few null values, + and their count is below the desired percentage of error records, DQOps will configure the data quality checks to raise data completeness issues for these columns. + +* [Data uniqueness](../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md) statistics for each column. + DQOps will use this information to configure duplicate detection checks for columns containing just a few duplicate values + that must be invalid or configure the distinct count checks to monitor the number of distinct values to stay in range. + + +### Data samples +DQOps configures several data quality checks using sample values captured from each column. +The types of data quality checks configured by the rule mining engine are listed below. + + * DQOps captures the top 100 most common values in each column. If the number of distinct values is below 100 and + the captured sample values cover all possible values in a column, DQOps will configure the + [values_found_in_set_percent](../categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md#how-to-validate-accepted-values-in-columns) dictionary checks. + If some values captured in the data samples are rare and present in less than the expected percentage of + erroneous rows (the default is 2% of records), DQOps will not include these values in the list of accepted values. + Running the data quality checks in the **accepted values** category will generate data quality issues, and the error samples will show these invalid values. + + When a data dictionary containing valid values is already defined in DQOps, and the majority of column values are present in any of + the [data dictionaries](../categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md#defining-data-dictionaries) defined by the user, + the rule mining engine will configure the **values in set** checks to use that dictionary. + This algorithm will configure data quality checks to detect values not present in the dictionary. + + * The sample values are also used to configure the [expected_texts_in_top_values](../categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md#testing-the-most-common-values) checks, + ensuring that the most common values are always the same in the data set over time. + + * Text values are analyzed for well-known patterns, such as [UUID identifiers](../examples/data-validity/percentage-of-valid-uuid.md), + [emails](../examples/data-validity/detect-invalid-emails.md), phone numbers, [ISO country codes](../examples/data-validity/percentage-of-valid-currency-codes.md), + or [currency codes](../categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md#validating-country-codes). + When DQOps detects columns mainly containing these values, relevant pattern validation checks are automatically configured. + + * DQOps will detect column values containing possible [patterns that can be analyzed with regular expressions](../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md#validating-custom-regular-expressions). + Suppose the sample column values contain texts of at least three components, such as numbers, texts, and special characters, such as `A/2024/11-C`. + In that case, DQOps will propose a regular expression pattern that will detect values that do not match the pattern. + + +### Results of data profiling checks +The rule mining engine is not limited to data quality checks that can be configured using sample values or data statistics. +Users can use the [data observability patterns](data-observability.md) (data quality policies) to configure +[data profiling checks](definition-of-data-quality-checks/data-profiling-checks.md) without any [data quality rules](definition-of-data-quality-rules.md) enabled. +These checks will only capture the [sensor readouts](../reference/parquetfiles/sensor_readouts.md) (the data quality metrics) without evaluating them with [data quality rules](definition-of-data-quality-rules.md) . + +For example, the user can enable all profiling checks in the +[PII (Personal Identifiable Information) check category](../categories-of-data-quality-checks/how-to-detect-pii-values-and-sensitive-data.md) +to detect columns containing a few sensitive values, such as emails or phone numbers inside comment columns. +When the rule mining engine notices that the [contains_email_percent](../checks/column/pii/contains-email-percent.md) +check finds 1% of records containing emails and the desired error rate is 2%, +DQOps will configure the [contains_email_percent](../checks/column/pii/contains-email-percent.md) check with a +[max_percent](../reference/rules/Comparison.md#max-percent) threshold 0%, identifying all these records as values containing a possible data leak. + +DQOps rule miner also supports custom data quality checks as long as they use built-in data quality rules, such as the [max_percent](../reference/rules/Comparison.md#max-percent) rule. + +The data quality results of the profiling checks are used further in DQOps when activating vetted data quality checks for continuous data quality monitoring +in the [monitoring](definition-of-data-quality-checks/data-observability-monitoring-checks.md) or [partition](definition-of-data-quality-checks/partition-checks.md) sections. + + +## Data quality check configuration steps +Follow these steps to configure the data quality lifecycle process in DQOps. + +### Basic data profiling +The first step of generating data quality rule configuration in DQOps, as part of a [data quality assessment process](data-quality-process.md#data-quality-assessment), +is [collecting basic data statistics](../working-with-dqo/collecting-basic-data-statistics.md). + +DQOps starts a statistics collection job for every table imported into DQOps using the user interface. +If the basic statistics are outdated or statistics collection has been stopped, +the user can click the **Collect statistics** button and wait until the screen shows the scale of null and distinct values for each column. + +![Basic data profiling statistics screen in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/profile-statistics-rule-mining-step-1-min.png){ loading=lazy; width="1200px" } + +### Advanced profiling +After the basic statistics and column value samples are collected, the rule mining screen can propose a configuration of data quality checks. +The most critical configuration parameter of the DQOps data quality rule miner is the desired error rate. It is a percentage of errors that can contain invalid values. + +DQOps will try to configure [data quality checks](definition-of-data-quality-checks/index.md) +in a way the data quality checks will detect data quality issues in 2% of records containing the most outstanding values. + +![Data quality rule mining screen with percentage of invalid records in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/propose-rules-rule-mining-step-2-min.png){ loading=lazy; width="1200px" } + +DQOps proposes the list of data quality checks instantly after entering the rule mining screen. +If any parameter configuration is changed, pressing the **Propose** button will generate a new proposal. + +The following screen shows how DQOps proposed the configuration of data quality checks. +DQOps supports changing the data quality check parameters and data quality rule thresholds before saving (applying) the proposed configuration, as shown below. + +![Review proposed data quality rule parameters and thresholds in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/review-proposed-thresholds-rule-mining-step-3-min.png){ loading=lazy; width="1200px" } + +### Managing data dictionaries +DQOps rule mining engine uses the list of most popular values in each text column to propose the configuration of `accepted values` checks. +The list of values does not need to be entered directly for each check. +DQOps supports using [data dictionaries](../categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md#defining-data-dictionaries), +which are stored as simple CSV files. +The list of valid values can be copied to a data dictionary and replaced with a reference to the data dictionary. + +![Define a data dictionary for a data quality category check in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/convert-to-dictionary-rule-mining-step-4-min.png){ loading=lazy; width="900px" } + +The data dictionary is created by clicking the **Convert to dictionary** button, entering the desired dictionary name, and clicking the **Save** button. + +![Save a data dictionary for a data quality category check in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/save-data-dictionary-rule-mining-step-5-min.png){ loading=lazy; width="600px" } + + +### Data quality check selection +If you disagree with the proposition of data quality check configuration, DQOps supports two ways of excluding them. + +The first option is to use the on/off switch to remove these checks from the proposal. +In that case, DQOps will propose these data quality checks the next time any user opens the rule mining screen. + +![Exclude a proposed data quality check in the DQOps rule miner](https://dqops.com/docs/images/concepts/data-quality-rule-mining/skip-selected-checks-rule-mining-step-6-min.png){ loading=lazy; width="1200px" } + +Another option is to disable the data quality check, which excludes it from execution and further propositions. +The red **stop** icon disables the check. + +![Disable a proposed data quality check in the DQOps rule miner](https://dqops.com/docs/images/concepts/data-quality-rule-mining/disable-selected-checks-rule-mining-step-7-min.png){ loading=lazy; width="1200px" } + + +### Data quality check evaluation +DQOps does not save the proposed configuration of data quality checks until the user presses the **Apply** button. + +![Apply proposed data quality checks in the DQOps rule miner](https://dqops.com/docs/images/concepts/data-quality-rule-mining/apply-checks-rule-mining-step-8-min.png){ loading=lazy; width="1200px" } + +Pressing the **Apply** button saves the configuration of data quality checks and their rules. +To verify the data quality health status of the table, data quality checks must be run. +DQOps will ask to confirm adding a data quality check evaluation job to the queue. + +![Run proposed data quality checks in the DQOps rule miner](https://dqops.com/docs/images/concepts/data-quality-rule-mining/run-checks-rule-mining-step-9-min.png){ loading=lazy; width="700px" } + +The progress of running the data quality checks can be tracked in the notification panel that shows the progress of running jobs in real time. +The **Run checks** job will be running as long as the circle icon highlighted below is visible. + +![Data quality run checks job status in DQOps after running proposed checks](https://dqops.com/docs/images/concepts/data-quality-rule-mining/run-checks-status-rule-mining-step-9-1-min.png){ loading=lazy; width="500px" } + + +### Data quality health review +After the **Run checks** job finishes, as shown in the screenshot below, it is possible to switch to the **Table quality status** screen, +which will show the summary of identified data quality issues. + +![Reviewing the data quality health status of tables after using the rule miner](https://dqops.com/docs/images/concepts/data-quality-rule-mining/finish-run-checks-rule-mining-step-10-min.png){ loading=lazy; width="1200px" } + +The **Table quality status** screen summarizes data quality issues identified for each column and each category of data quality checks. +Switching to the [data quality dimension](data-quality-dimensions.md) view is possible using a switch control at the top of the screen. + +DQOps calculates a [data quality KPI score](definition-of-data-quality-kpis.md), measured as the percentage of data quality checks that passed successfully. + +In order to review the data quality issues, you can expand the cell to learn which data quality check failed or click on the name of the column +to navigate to a detailed [data quality check editor](dqops-user-interface-overview.md#check-editor) screen. + +![Reviewing the data quality health status of tables and columns after using the rule miner](https://dqops.com/docs/images/concepts/data-quality-rule-mining/review-profiling-status-rule-mining-step-11-min.png){ loading=lazy; width="1200px" } + + +### Data quality issue review +DQOps will [collect error samples](data-quality-error-sampling.md) of invalid values that did not pass the data quality check evaluation. +In the following example, the rule mining engine configured a desired list of column values, but the `Clothing Sets` value was scarce and was present in less than 2% of records. +Because the desired percentage of invalid rows was 2%, DQOps did not add this value to the list of accepted values, assuming it is invalid. + +The list of invalid values can be reviewed on the **Error sampling** tab in the data quality check editor. +If this value is valid, the list of `expected_values` should be updated in the editor to include that value. + +![Reviewing detected data quality issues in DQOps using the data quality error sampling module](https://dqops.com/docs/images/concepts/data-quality-rule-mining/review-error-samples-rule-mining-step-12-min.png){ loading=lazy; width="1200px" } + + +### Disabling false-positive checks +The rule mining editor can also propose the configuration of checks that will generate false-positive results. +These issues are incorrectly detected due to an aggressive configuration of the desired percentage of invalid records. +The best option for handling these checks after reviewing the error samples is disabling them. +You should click the **Save** button at the top of the screen to store any configuration changes. + +![Disabling false-positive data quality checks in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/disable-false-positive-checks-rule-mining-step-13-min.png){ loading=lazy; width="1200px" } + + +## Rule mining parameters +This section describes the configuration parameters shown at the top of the rule mining screen. + +### Propose checks for a column +It is possible to propose the configuration of data quality checks for only one column or several columns containing the same text inside the column name. +The following example shows how to set the column name. +The rule engine will generate a new proposition of check configuration after clicking the **Propose** button. + +![Use data quality rule miner engine in DQOps to propose checks for selected columns](https://dqops.com/docs/images/concepts/data-quality-rule-mining/propose-checks-rule-mining-min.png){ loading=lazy; width="1200px" } + +The table had eight columns containing the word "product". + +![Data quality checks proposed for a subset of columns in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/propose-checks-for-column-rule-mining-min.png){ loading=lazy; width="1200px" } + +### Rule miner parameters +The checkboxes are divided into two groups. The first group of fields controls how the rule mining engine handles already configured data quality checks. + +These parameters become very important when using the rule engine later to convert the [profiling checks](definition-of-data-quality-checks/data-profiling-checks.md) +to [table monitoring](definition-of-data-quality-checks/data-observability-monitoring-checks.md) and +[partition monitoring checks](definition-of-data-quality-checks/partition-checks.md), which is described later in this guide. + +| Parameter name | Description | +|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Copy failed profiling checks | Copy the configuration of profiling checks that failed during the last execution. The preferred approach is to review the profiling checks, disable false-positive checks, and enable this configuration to copy the reviewed checks to the monitoring and partitioned checks for continuous monitoring. | +| Copy disabled profiling checks | Copy the configuration of disabled profiling checks. This option is effective for monitoring or partitioned checks only. By default it is disabled, leaving failed or incorrectly configured profiling checks only in the profiling section to avoid decreasing the [data quality KPI](definition-of-data-quality-kpis.md). | +| Copy profiling checks | Copy the configuration of enabled profiling checks to the monitoring or partitioned checks. This option is effective for monitoring or partitioned checks only. By default it is enabled, allowing to migrate configured profiling checks to the monitoring section to enable Data Observability of these checks. | +| Tune quality policy checks | Reconfigure the rule thresholds of data quality checks that were activated using [data observability](data-observability.md) rule patterns (data quality policies). | + + +### Data quality checks +The selection of data quality checks the rule mining engine will configure is managed by checkboxes below the horizontal line. + +| Data quality checks | Description | +|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Minimum row count](../categories-of-data-quality-checks/how-to-detect-data-volume-issues-and-changes.md#minimum-row-count) | Configure the minimum row count based on the basic statistics captured from the table. | +| [Expected columns count](../categories-of-data-quality-checks/how-to-detect-table-schema-changes.md) | Configure a table schema check that ensures that the count of column stays the same as the count of columns detected during data profiling. | +| [Column exists](../checks/column/schema/column-exists.md) | Configure a column exists check for each column to report when the column is no longer present. | +| [Timeliness checks](../categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md) | Configure all types of table's timeliness checks (freshness, staleness, ingestion delay). This option works only when the table is correctly configured to be analyzable by timeliness check, and has the timestamp column selected in the configuration of the table in the Data sources section. | +| [Nulls (prohibit nulls)](../categories-of-data-quality-checks/how-to-detect-empty-or-incomplete-columns-with-nulls.md#incomplete-columns) | Configure data quality checks that detect columns that have some null values in columns. When the percentage of null columns is below the value of the 'Error rate (% rows)' field, DQOps will raise a data quality issue when any null values are detected. | +| [Not nulls (require nulls)](../categories-of-data-quality-checks/how-to-detect-empty-or-incomplete-columns-with-nulls.md#detect-a-minimum-number-of-non-null-values) | Configure data quality checks that require that columns already containing null values will contain some null values. This option could be desirable in some rare cases. | +| [Detected datatype in texts](../categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-text-fields.md) | Configure a check that verifies all values in a text column and detects a common data type. If all column values matched the same data type, such as integers, floats, dates or timestamps, DQOps will configure a check that will monitor if any values not matching that data type appear in the column. This check is usable for raw tables in the landing zone. | +| [Uniqueness checks](../categories-of-data-quality-checks/how-to-detect-data-uniqueness-issues-and-duplicates.md) | Configure uniqueness checks that detect columns with just a few duplicate values, and columns with a relatively low number of distinct values. DQOps will monitor if duplicate values appear, or the number of distinct values increases. | +| [Numeric values ranges](../categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-numeric-fields.md) | Validate the values in numeric columns to detect if the values fall within the ranges that were observed during data profiling. DQOps will try to configure the *min*, *max* and *mean' checks. | +| [Median and percentile ranges](../checks/column/numeric/median-in-range.md) | Propose the default configuration of the median and percentile in range checks that validate the value at a given percentile, such as a value in middle of all column values. The default value of this parameter is 'false' because calculating the median requires running a separate query on the data source, which is not advisable for data observability. | +| [Text length ranges](../categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-text-fields.md) | Configure the checks that analyze text statistics of text columns, such as the minimum and maximum length of text values. | +| [Word count in range](../checks/column/text/min-word-count.md) | Propose the default configuration of the minimum and maximum word count of text columns. DQOps applies this checks when the minimum and maximum row count is at least 3 words. | +| [Percent of true/false](../categories-of-data-quality-checks/how-to-detect-data-quality-issues-in-bool-fields.md) | Configure the checks that analyze boolean values and will configure a range percent check for the less common value (true or false). | +| [Dates out of range](../categories-of-data-quality-checks/how-to-detect-invalid-dates.md) | Configure the checks that detect invalid dates that are far in the past, or far in the future. | +| [Values in a set (dictionary)](../categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md#verify-that-only-accepted-values-are-used) | Proposes the configuration the categorical values checks `value_in_set_percent` from the `accepted_values` category. These checks will be configured to ensure that the column contains only sample values that were identified during data profiling. | +| Treat rare values as invalid | Configure the `value_in_set_percent` checks from the `accepted_values` category to raise data quality issues for rare values. DQOps will not add rare categorical values to the list of expected values. | +| [Texts in top values](../categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md#testing-the-most-common-values) | Propose the configuration the `texts_in_top_values` check from the `accepted_values` category. This check find the most common values in a table and ensures that they are the same values that were identified during data profiling. | +| [Texts parsable to data types](../categories-of-data-quality-checks/how-to-verify-text-values-are-parsable.md) | Proposes the configuration the data type conversion checks that verify if text values can be converted to more specific data types such as boolean, date, float or integer. This type of checks are used to verify raw tables in the landing zones. | +| [Standard text patterns](../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md#built-in-patterns) | Propose the default configuration for the patterns check that validate the format of popular text patterns, such as UUIDs, phone numbers, or emails. DQOps will configure these data quality checks when analyzed columns contain enough values matching a standard pattern. | +| [Detect regular expressions](../categories-of-data-quality-checks/how-to-detect-bad-values-not-matching-patterns.md#validating-custom-regular-expressions) | Analyze sample text values and try to find a regular expression that detects valid values similar to the sample values. | +| [Whitespace checks](../categories-of-data-quality-checks/how-to-detect-blank-and-whitespace-values.md) | Propose the default configuration for the whitespace detection checks. Whitespace checks detect common data quality issues with storing text representations of null values, such as `null`, `None`, `n/a` and other texts that should be stored as regular *NULL* values. | +| [Apply PII check rules](../categories-of-data-quality-checks/how-to-detect-pii-values-and-sensitive-data.md) | Applies rules to Personal Identifiable Information checks (sensitive data), but only when the checks were activated manually as profiling checks, or through a data quality check pattern that scans all columns for PII data. | +| [Custom checks](../working-with-dqo/creating-custom-data-quality-checks.md) | Custom data quality checks must use DQOps built-in data quality rules, such as max_percent, min_percent or max_count to find invalid values. | + + +## Data quality monitoring +The next phase after performing the [data quality assessment](data-quality-process.md#data-quality-assessment) is +configuring [data observability](data-quality-process.md#data-quality-monitoring), which is a continuous monitoring of data quality. + +DQOps will use its [CRON scheduler](../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md) to evaluate the data quality daily +and [raise incidents](data-quality-process.md#data-quality-incident-management) for identified data quality issues. + +!!! tip "Enabling reviewed data quality checks for continuous data quality monitoring" + + The principal idea behind DQOps is its two-stage data quality check review process. + In the first stage, users should perform the data quality assessment by [configuring data profiling checks](#advanced-profiling) to [identify data quality issues](#data-quality-health-review). + + The second step is the review of [false-positive data quality checks](#disabling-false-positive-checks). + The checks that are incorrectly configured should be [disabled](#disabling-false-positive-checks), as shown before. + + Once the list of active data profiling checks is limited only to approved checks that are passing or detecting acknowledged data quality issues, + you should switch to the **Monitoring checks** section of DQOps and use the rule mining screen again to copy the configuration of checks + from the **Profiling** section to the **Monitoring checks** section, to allow DQOps continuously evaluate only valid checks. + + The rule mining screen on the **Monitoring checks** and **Partition** sections works slightly differently and copies the already configured + data quality checks and their parameters from the DQOps data profiler found in the **Profiling** section of the user interface. + + +### Configuring monitoring checks +The monitoring checks in DQOps perform regular full-table scans and analyze small to medium-sized tables or tables in which any row could be updated, such as a price list table. + +Pressing the **Apply** button will save the configuration of [monitoring checks](definition-of-data-quality-checks/data-observability-monitoring-checks.md) +and run the data quality checks to save the first data quality health checkpoint. + +![Configuring full-table scan data quality checks in DQOps data monitoring section](https://dqops.com/docs/images/concepts/data-quality-rule-mining/apply-monitoring-checks-rule-mining-step-14-min.png){ loading=lazy; width="1200px" } + +The data quality check result captured during the first check execution is shown in the screenshot below. +DQOps will store one checkpoint per day for daily monitoring checks. + +![Reviewing data quality checkpoints of table monitoring checks in DQOps](https://dqops.com/docs/images/concepts/data-quality-rule-mining/review-monitoring-checkpoint-rule-mining-step-15-min.png){ loading=lazy; width="1200px" } + + +### Configuring partition checks +The generation of [partition checks](definition-of-data-quality-checks/partition-checks.md) that validate partitioned data is similar to configuring [monitoring checks](#configuring-monitoring-checks). +Please read the [partition check description](definition-of-data-quality-checks/partition-checks.md) because DQOps requires additional configuration to enable these types of checks. +Partition checks require selecting the date or timestamp column used for [time partitioning](definition-of-data-quality-checks/partition-checks.md#setting-up-date-partitioning-column). + + +## What's next + +- Get familiar with the [DQOps user interface](dqops-user-interface-overview.md) +- Learn how to [connect data sources](configuring-data-sources.md) +- Learn how to [configure data quality checks](configuring-data-quality-checks-and-rules.md) diff --git a/docs/dqo-concepts/definition-of-data-quality-checks/index.md b/docs/dqo-concepts/definition-of-data-quality-checks/index.md index ee746191d0..3ceefc2076 100644 --- a/docs/dqo-concepts/definition-of-data-quality-checks/index.md +++ b/docs/dqo-concepts/definition-of-data-quality-checks/index.md @@ -210,6 +210,7 @@ stored in the [*$DQO_USER_HOME/checks/\*\*/\*.dqocheck.yaml*](../../reference/ya - Learn how to [configure data sources](../configuring-data-sources.md) - Learn how to [configure data quality checks](../configuring-data-quality-checks-and-rules.md) - Learn how to [run data quality checks](../running-data-quality-checks.md) +- Data quality checks do not need to be configured manually. Learn how the [data quality rule mining](../data-quality-rule-mining.md) engine can automatically propose and configure data quality checks to detect the most common data quality issues. - [Learn more about profiling checks](data-profiling-checks.md) - [Learn more about monitoring checks](data-observability-monitoring-checks.md) - [Learn more about partition checks](partition-checks.md) diff --git a/docs/dqo-concepts/dqops-user-interface-overview.md b/docs/dqo-concepts/dqops-user-interface-overview.md index 286984da2d..8a7fb69a9c 100644 --- a/docs/dqo-concepts/dqops-user-interface-overview.md +++ b/docs/dqo-concepts/dqops-user-interface-overview.md @@ -89,7 +89,7 @@ The **Refresh** button located at the top right of the screen, allows you to ref At the top of The Incidents summary screen, there are two sections displaying the summaries of **Open** and **Acknowledged** incidents. Each section is divided into three blocks based on severity level: Warnings, Errors and Fatal errors. In each block, -there is a summary of new incidents from the last 2 months along with detailed data showing the number of incidents detected in the last 24h, last 7 days, +there is a summary of new incidents from the last 2 months, along with detailed data showing the number of incidents detected in the last 24h, last 7 days, current month and previous month. Below the **Open** and **Acknowledged** incidents summaries, there is a table that list incidents grouped by the selected check category or the quality dimension, depending on your grouping selection. @@ -248,10 +248,13 @@ In the **Data Source** section, you can access: - **Schedule:** Allows you to [configure of the check execution schedule](../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md) at the connection level. - **Comments:** Allows adding comments to your connection. - **Labels:** Allows adding labels to your connection. -- **Tables:** Displays a summary of the data quality status for tables in this connection. -- **Columns:** Displays a summary of the data quality status for columns in this connection. +- **Schemas:** Allow importing schemas and tables. +- **Data quality summary:** Displays summaries of the data quality status. You have the option to choose one of two subtabs: + - **Tables:** This subtab provides a summary of the data quality status for tables in this connection. + - **Columns:** This subtab provides a summary of the data quality status for columns in this connection. - **Default grouping template:** Allows setting up data grouping globally at the data source level. [Learn how to configure data grouping](../working-with-dqo/set-up-data-grouping-for-data-quality-checks.md). -- **Incidents and notifications:** Allows configuring incidents and Webhooks for notifications. [Learn more about incidents](../working-with-dqo/managing-data-quality-incidents-with-dqops.md) that let you keep track of the issues that arise during data quality monitoring. [Learn how to configure notifications](../integrations/webhooks/index.md) whenever a new incident is created or modified. +- **Incident grouping**: Allows configuring incidents grouping level. [Learn more about incidents](../dqo-concepts/grouping-data-quality-issues-to-incidents.md) that let you keep track of the issues that arise during data quality monitoring. +- **Notifications:** Allows configuring incidents and Webhooks for notifications. [Learn how to configure notifications](../dqo-concepts/grouping-data-quality-issues-to-incidents.md#configure-notification-for-an-incident) whenever a new incident is created or modified. ![Main workspace tabs - Profiling](https://dqops.com/docs/images/working-with-dqo/navigating-the-graphical-interface/main-workspace-tabs-profiling2.png){ loading=lazy; width="1200px" } @@ -316,7 +319,7 @@ If you select the "Multiple levels" option, you will switch to the Advanced chec In the advanced mode, you can configure different thresholds for warning, error, and fatal error severity levels. -To access the Advanced mode, select the "Multiple levels" option from the Issue severity level dropdown menu in +To access the Advanced mode, select the **Multiple levels** option from the Issue severity level dropdown menu in the simplified data check editor, as shown above. Once you have made this selection, you will receive a prompt informing you that the editor will switch to an advanced mode. @@ -362,7 +365,7 @@ The buttons and icons located on the left side of each check name allow you to p - **Check Information:** View tooltip with detailed information about the specific check. - **Check results colored square:** Indicates the result of the last check run: - - Green for a valid result + - Green for a correct result - Yellow for a warning - Orange for an error - Red for a fatal error @@ -388,7 +391,7 @@ dimension, depending on the selected option. The first row displays the results on the table, while the following rows show the column names and the results from the executed column-level checks. The color indicate the current or the highest severity status: -- Green for a valid result +- Green for a correct result - Yellow for a warning - Orange for an error - Red for a fatal error @@ -423,7 +426,7 @@ If the job is queued or running you can cancel or stop it by clicking the X butt ### **Run checks summary** You can view the result of executed checks by simply hovering on run checks job. Color of the square represents the highest -severity status of the results (green for a valid result, yellow for a warning, orange for an error and red for a fatal error). +severity status of the results (green for a correct result, yellow for a warning, orange for an error and red for a fatal error). ![DQOps job results](https://dqops.com/docs/images/working-with-dqo/navigating-the-graphical-interface/job-results.png) diff --git a/docs/dqo-concepts/grouping-data-quality-issues-to-incidents.md b/docs/dqo-concepts/grouping-data-quality-issues-to-incidents.md index ecf0ede8d2..7ffacc5b6e 100644 --- a/docs/dqo-concepts/grouping-data-quality-issues-to-incidents.md +++ b/docs/dqo-concepts/grouping-data-quality-issues-to-incidents.md @@ -54,9 +54,9 @@ The following statuses are used in the data quality incident workflow. ## Grouping issues into incidents -The **data quality incident** grouping level is configured on a connection level as shown on the *Incidents and Notifications* screen below. +The **data quality incident** grouping level is configured on a connection level as shown on the *Incidents grouping* screen below. -![Incidents And Notifications tab](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incidents-and-notifications-settings2.png){ loading=lazy; width="1200px" } +![Incidents And Notifications tab](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incident-grouping.png){ loading=lazy; width="1200px" } The following grouping levels are supported: @@ -112,6 +112,8 @@ but the support team decided that they will not be resolved. ## Incident management +### **Overview** + DQOps allows you for a quick overview of all incidents within your environment. To access the **Incidents summary**, click on the DQOps logo in the top left corner and select the **Incidents summary** tab. @@ -138,7 +140,7 @@ When a single incident is clicked, DQOps shows the incident detail screen. The u to the clipboard and send to another DQOps user, who can review the issue. Also, the link is shown in the [Slack incident notifications](../integrations/slack/configuring-slack-notifications.md). -![Incident details screen](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incident-details-screen3.png){ loading=lazy; width="1200px" } +![Incident details screen](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incident-details-screen4.png){ loading=lazy; width="1200px" } The incident management screens are described in details on the [working with incidents and notifications](../working-with-dqo/managing-data-quality-incidents-with-dqops.md) page. @@ -151,7 +153,7 @@ useful when you are actively working to resolve the underlying issue. To disable a check, click on the "Disable check" button in the top right corner of the Incident details screen. Once confirmed, the check will be temporarily stopped from executing. -![Disabling check](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/disabling-check-button.png){ loading=lazy; width="1200px" } +![Disabling check](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/disabling-check-button2.png){ loading=lazy; width="1200px" } You can verify that the check has been disabled by navigating to the check editor screen. @@ -166,7 +168,7 @@ This can be helpful in situations where the check might be overly sensitive. Clicking the **Reconfigure** button will decrease the rule threshold for the data quality check that caused the incident by 30%. For more significant adjustments, you can click the **Recalibrate** button multiple times. Each click will further reduce the check's thresholds by an additional 30%. -![Reconfigure check button](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/recalibrate-check-button.png){ loading=lazy; width="1200px" } +![Reconfigure check button](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/recalibrate-check-button2.png){ loading=lazy; width="1200px" } The following example YAML files illustrate the `daily_partition_row_count` check configuration before and after reconfiguration. Notice that the `min_count` rule has been reduced from 1000, to 700. @@ -217,12 +219,214 @@ Notice that the `min_count` rule has been reduced from 1000, to 700. ``` +### **Configure notification for an incident** + +To receive notifications for specific data quality incidents, you can create or edit notification filters. +To do this, click on the envelope icon located in the top right corner of the incident details screen. + +![Configure notification filter](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configure-notificaiton-filter.png){ loading=lazy; width="1200px" } + +The action after clicking on the envelope icon will depend on whether the incident matches any existing incident. +[notification filters](grouping-data-quality-issues-to-incidents.md#notification-filters). + +**Adding a new notification filter**. + +If no notification has ever been configured for the incident, you will see a popup message informing you to create a new notification configuration. + +![Configure incident popup](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configure-incident-popup.png){ loading=lazy; width="600px" } + +Once approved, you will be redirected to the page to create the new incident notification filter. +The configuration form will be partially filled based on the incident's data. The details of the filter configuration can be +found in the section [Notification filters](grouping-data-quality-issues-to-incidents.md#notification-filters) below. + +![New notification filter](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/new-notification-filter.png){ loading=lazy; width="1200px" } + +**Editing an existing notification filter** + +If the notification matches an existing filter, a screen with this filter will open, and you can edit the filter's settings. + + ## Incident notifications -DQOps supports automation of the incident workflows by using notifications. -Please read the description of integrating incidents with other systems using [webhooks](../integrations/webhooks/index.md). -Incident notifications configured for [Slack](../integrations/slack/configuring-slack-notifications.md) will -use formatted messages that are shown on the Slack channels as shown below. +Incident notifications allow you to receive alerts via email or webhook whenever a new data quality incident occurs or its status changes. + +There are two levels of notification configuration in DQOps: + +1. **Global notifications:** This configuration applies to all incidents unless overridden. You can access it through the Configuration section > Global incident notifications. +2. **Connection notifications:** These configurations take precedence over global notifications and are specific to individual data sources. + You can access them by navigating to a specific data source under the Data sources section > [Connection name] > Notifications tab. + +**Default notification configuration** + +The default notification is used as a backup when no filters match the incident. You can configure it within both global and connection notifications. + +Initially, the default configuration is empty. Notifications will be sent to the specified addresses when they are set +in the **Addresses for notifications of an incident state change** section. + +If the incident **does not match** any filter and no address is set in the default notification, **no notification will be sent**. + +### Configuring Global notifications + +To access the **Global notifications** configuration: + +1. Click on the **Configuration** section +2. Select the **Global incident notifications** in the tree view on the left. + +If no notifications filters are configured, only the default notification configuration will be present. +Here you can also create a new [notification filter](grouping-data-quality-issues-to-incidents.md#notification-filters) +by clicking on the **Add notification filter** button in the top right corner. + +![Global incident notifications](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/global-incident-notifications.png){ loading=lazy; width="1200px" } + +To configure the **default** notification, open the default notification configuration by clicking the **default** name or by clicking the **Edit** action button on the right. + +![Global incident notifications default](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/global-incident-notification-default.png){ loading=lazy; width="1200px" } + +Fill in the **email address** or **webhook URL** for the appropriate status fields. + +You can also use multiple email addresses separated by the comma character (,). Mixing email addresses with webhook addresses is also allowed if you use commas between them. + +The address field for the specific status can also be left as empty so no notifications will be received for that incident status. + +### Configuring Connection notifications configuration +To access the **Connection notifications** configuration: + +1. Click on the **Data sources** . +2. Select the connection of choice from the tree view on the left. +3. Click on the **Notification** tab in the right panel. + +If no notifications filters are configured, only the default notification configuration will be present. +Here you can also create a new [notification filter](grouping-data-quality-issues-to-incidents.md#notification-filters) by clicking on the **Add notification filter** button. + +![Connection notifications configuration](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/connection-notification-configuration.png){ loading=lazy; width="1200px" } + +To configure the **default** notification, open the default notification configuration by clicking the **default** name or the **Edit** action button on the right. +Fill in the **email address** and/or **webhook URL** to the appropriate status fields, as described above. + +### Notification filters + +You have the option to further refine notifications by setting filters based on various criteria. Filters can be added to both Global and Connection notifications. + +To add a new filter on the Global or Connection configuration: + +1. Navigate to the Global or Connection configuration. +2. Click the **Add notification filter button**. + + ![Add filter to global incident notifications](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/global-incident-notifications-add-filter.png){ loading=lazy; width="1200px" } + +3. Set the name for the notification filter and specify the field that will be filtered. + + ![Filtered notifications](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/filtered-notification.png){ loading=lazy; width="1200px" } + +4. Fill in any field in the **Addresses for notifications of an incident state change** section with **email address** and/or **webhook URL**. + + +The filter configuration allows to specify the following: + +| Field name | Description | +|---------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Notification name** | The unique name of the filtered notification. | +| **Priority** | The priority of the notification. The notifications with a smaller number have higher priority and are matched first. | +| **Notification message** | Additional message to be attached to the notification message. This can be the description of the notification, SLA note, etc. | +| **Disabled** | Whether to send notifications for this filter. | +| **Process additional notification filters** | Specifies whether the next filters in priority order should be taken into account. When not set, the notification will be sent only to the addresses that match the first notification filter in the priority order. | +| **Do not create incidents** | Specifies whether the notification message should be sent when the notification matches the filters. Setting this flag to true can break searching for the next matching notification without sending the notification message. | +| **Connection** | The target connection name filter. Supports search patterns in the format: 'source\*', '\*_prod', 'prefix\*suffix'. | +| **Schema** | The target schema name filter. Supports search patterns in the format: 'schema_name_\*', '\*_schema', 'prefix\*suffix'. | +| **Table** | The target table name filter. Supports search patterns in the format: 'table_name_\*', '\*table', 'prefix\*suffix'. | +| **Data group name** | The target data group name filter. Supports search patterns in the format: 'group_name_\*', '\*group', 'prefix\*suffix'. | +| **Check name** | The target check name filter. Uses the short check name which is the name of the deepest folder in the *checks* folder. This field supports search patterns such as: 'profiling_\*', '\*_count', 'profiling_\*_percent'. | +| **Highest severity** | The target highest severity filter. | +| **Table priority** | The target table priority filter. | +| **Quality dimension** | The target quality dimension filter. | +| **Check category** | The target check category filter, for example: *nulls*, *volume*, *anomaly*. |*string*| | +| **Check type** | The target check type filter. One of: profiling, monitoring, or partitioning. | + +Setting a filter field (e.g. table) means that the incident's table name has to match the filter to receive the notification message. +When other filters are left empty, they are not taken into account and only non-empty filters are verified. + +Filters such as connection, schema, table, data group name and check name support search patterns in format prefix\*, \*suffix, prefix\*suffix. + +### Multiple notification filters + +You can create multiple notification filters within a single configuration for specific scenarios. + +For example, a Data Engineering Team might receive notifications about Timeliness (data delay) or Validity (data ingestion issues) issues, +while a Data Asset Manager gets alerted about market completeness problems. + +When using multiple filtered notification configuration, DQOps will send the notification based on the priority in **ascending order**, +unless the **Process additional notification filters** checkbox is selected. + +Selecting **Process additional notification filters** checkbox will prevent the first matching filter from interrupting +the search for the next configured filters. The system will continue to search for the next match. +If this option is not selected, the first filter will interrupt the search process. + +The notifications are sent using all the matched notification filters in this process. To stop the search process for +the next matching filter without sending the notification message, select the **Do not create incidents** option. +All preceding notifications that match the filters will be sent until the interrupting filter is matched. + +### Email notifications + +The notification received through email contains the following information about the incident. + +| Field name | Description | +|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Full table name** | Schema and table name affected by a data quality incident. The name is linked to the incident in the application. | +| **Data source** | Connection name affected by a data quality incident. | +| **Last seen** | The UTC timestamp when the data quality incident was last seen. Appears for all incident statuses except for open. | +| **Quality dimension** | The data quality dimension affected by a data quality incident. | +| **Check category** | The data quality check category affected by a data quality incident. | +| **Highest severity** | The highest severity of the failed check detected in this data quality incident. Possible values include warning, error, and fatal. | +| **Failed checks count** | The total number of failed data quality checks that were seen when the incident was raised for the first time. | +| **Table priority** | Shown when present. Table priority of the table that was affected by a data quality incident. | +| **Issue url** | Shown when present. The link (URL) to a ticket in an external system that is tracking this incident. | +| **Data group name** | Shown when present. The data group name that was affected by a data quality incident. | +| **Check type** | Shown when present. The check type that was affected by a data quality incident. | +| **Check name** | Shown when present. The check name that was affected by a data quality incident. | +| **Message** | Shown when present. The additional message of the notification message configured by the user from the filtered notifications. Default notification does not contain such a field. | + + +A sample notification received via email is shown below. + +![Email notification](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/email-notification.png){ loading=lazy; width="600px" } + + +### SMTP Server configuration + +Email notifications require configuring an SMTP server to send email messages. + +The SMTP server can be configured by using **environment variables** or by setting **local settings in the user's home directory**. + +The required environment variables are the following: + +| Environment variable | Description | +|------------------------------|------------------------------| +| **DQO_SMTP_SERVER_HOST** | The SMTP Server host address | +| **DQO_SMTP_SERVER_PORT** | The port for SMTP service | +| **DQO_SMTP_SERVER_USESSL** | Use SSL flag (true / false) | +| **DQO_SMTP_SERVER_USERNAME** | User name | +| **DQO_SMTP_SERVER_PASSWORD** | Password | + + +The example of the SMTP Server configuration from the **.localsettings.dqopsettings.yaml** file + +```yaml +spec: + smtp_server_configuration: + host: + port: + use_ssl: + username: + password: +``` + +### Webhooks + +DQOps supports the automation of incident workflows by utilizing notifications. +Review the description of how incidents can be integrated with other systems using [webhooks](../integrations/webhooks/index.md). + +Incident notifications configured for [Slack](../integrations/slack/configuring-slack-notifications.md) +use formatted messages that are displayed on the Slack channels as shown below. ![slack-message-open](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configuring-slack-notifications/slack-message-open.png) diff --git a/docs/dqo-concepts/index.md b/docs/dqo-concepts/index.md index a63793bd69..c76491b2a4 100644 --- a/docs/dqo-concepts/index.md +++ b/docs/dqo-concepts/index.md @@ -7,6 +7,18 @@ Follow this guide to learn each concept of DQOps Data Quality Operations Center ## List of DQOps concepts This article is a dictionary of DQOps terms. Click on the links to learn about every concept. +### **[Data quality process](data-quality-process.md)** +DQOps follows a two-stage data quality process. The first step is a [data quality assessment](data-quality-process.md#data-quality-assessment) using the [data profiler](definition-of-data-quality-checks/data-profiling-checks.md) . +This step identifies confirmed data quality issues. In the second stage, +users configure [monitoring](definition-of-data-quality-checks/data-observability-monitoring-checks.md) and [partition checks](definition-of-data-quality-checks/partition-checks.md) that regularly verify data quality using Data Observability. + + +### **[Data quality rule miner](data-quality-rule-mining.md)** +The data quality rule miner is responsible for proposing the configuration of +[data quality checks](definition-of-data-quality-checks/index.md) and their +[rule thresholds](definition-of-data-quality-rules.md) to detect [the most common data quality checks](data-quality-rule-mining.md#information-used-for-proposing-checks). + + ### **[What is a data quality check](definition-of-data-quality-checks/index.md)** A data quality check detects data quality issues. The check in DQOps is defined as a pair of a [sensor](definition-of-data-quality-sensors.md) that captures metrics from the data source and a [rule](definition-of-data-quality-rules.md) that verifies the sensor's readout. @@ -28,6 +40,11 @@ DQOps has three types of data quality checks: incrementally any table that has a date column. +### **[User interface overview](dqops-user-interface-overview.md)** +The user interface in DQOps is using a tabbed application that resembles many popular database management tools. +Configuring data quality checks on multiple tables at the same time is supported in separate tabs. + + ### **[Configuring data sources](configuring-data-sources.md)** DQOps stores the data source configuration and the metadata of imported tables in YAML files. The files are stored in the current folder, called the [`DQOps user home`](dqops-user-home-folder.md). @@ -67,6 +84,11 @@ Data quality checks configured for each table and column are executed by targeti check name, check type, check category or even labels assigned to tables or columns. +### **[Data quality rule mining](data-quality-rule-mining.md)** +DQOps supports data quality rule mining, a feature that uses statistics and sample data to automatically configure data quality checks, +which will detect the most common data quality issues without manual configuration. + + ### **[Data observability](data-observability.md)** When a new data source is imported into DQOps, a default set of data quality checks is activated. The main purpose of data observability is to detect anomalies in the data, database downtimes, schema changes, @@ -167,11 +189,6 @@ Data grouping allows detecting data quality issues for groups of rows loaded by different data pipelines, or received from different vendors or departments. -### **[User interface overview](dqops-user-interface-overview.md)** -The user interface in DQOps is using a tabbed application that resembles many popular database management tools. -Configuring data quality checks on multiple tables at the same time is supported in separate tabs. - - ### **[Command-line interface](command-line-interface.md)** Command-line access to DQOps is supported by a shell interface. The DQOps shell supports command and table name completion. diff --git a/docs/dqo-concepts/running-data-quality-checks.md b/docs/dqo-concepts/running-data-quality-checks.md index 35af359588..453d1bec70 100644 --- a/docs/dqo-concepts/running-data-quality-checks.md +++ b/docs/dqo-concepts/running-data-quality-checks.md @@ -554,7 +554,7 @@ Then take the key and use it as the token when creating an `AuthenticatedClient` ``` { .python linenums="1" } from dqops import client -dqops_client = client.AuthenticatedClient(base_url="http://localhost:8888", token="Your DQO API Key") +dqops_client = client.AuthenticatedClient(base_url="http://localhost:8888", token="Your DQOps API Key") ``` Now you can call operations on DQOps. The following code shows how to execute data quality checks diff --git a/docs/dqo-concepts/types-of-data-quality-dashboards.md b/docs/dqo-concepts/types-of-data-quality-dashboards.md index d7c3aadb5a..3671008580 100644 --- a/docs/dqo-concepts/types-of-data-quality-dashboards.md +++ b/docs/dqo-concepts/types-of-data-quality-dashboards.md @@ -30,7 +30,7 @@ to the data quality lakehouse. ## Navigating dashboards -To view dashboards, simply go to the **Data Quality Dashboard** section, and select the dashboard of interest from the tree +To view dashboards, simply go to the **Data quality dashboard** section, and select the dashboard of interest from the tree view on the left. There are several groups and subgroups of data quality dashboards dedicated to analyzing results from data quality checks. diff --git a/docs/dqops-installation/install-dqops-using-pip.md b/docs/dqops-installation/install-dqops-using-pip.md index 7462a4d5b7..7f6c0ae25e 100644 --- a/docs/dqops-installation/install-dqops-using-pip.md +++ b/docs/dqops-installation/install-dqops-using-pip.md @@ -4,7 +4,7 @@ This guide shows how to quickly download DQOps from PyPi, and start a local deve ## Overview DQOps is available as a Python package [dqops](https://pypi.org/project/dqops/). This package contains a bootstrapper that -will download a full DQOps distribution from [DQO GitHub](https://github.com/dqops/dqo) and install a Java JRE 17 in a version +will download a full DQOps distribution from [DQOps GitHub](https://github.com/dqops/dqo) and install a Java JRE 17 in a version specific to the current platform. DQOps runs on Windows, Linux and MacOS. Both x64 and arm8 platforms are supported. !!! note "Running DQOps as a server" diff --git a/docs/examples/data-accuracy/integrity-check-between-columns-in-different-tables.md b/docs/examples/data-accuracy/integrity-check-between-columns-in-different-tables.md index ac9895fec4..f65aa1f6b6 100644 --- a/docs/examples/data-accuracy/integrity-check-between-columns-in-different-tables.md +++ b/docs/examples/data-accuracy/integrity-check-between-columns-in-different-tables.md @@ -129,7 +129,7 @@ The Sensor readouts category displays the values obtained by the sensors from th The Execution errors category displays any error that occurred during the check's execution. The actual value in this example is 100%, which is above the minimum threshold level set in the warning (99.0%). -The check gives a valid result (notice the green square to the left of the check name). +The check gives a correct result (notice the green square to the left of the check name). ### **Synchronize the results with the cloud account** @@ -227,7 +227,7 @@ check run ``` Access the results which should be similar as the one below. -The percentage of matching values in the `state_fips_code` column is above the 99% and the check shows valid result. +The percentage of matching values in the `state_fips_code` column is above the 99% and the check shows correct result. ``` Check evaluation summary per table: diff --git a/docs/examples/data-availability/detect-table-availability-issues.md b/docs/examples/data-availability/detect-table-availability-issues.md index a75d705f3e..278caf5b29 100644 --- a/docs/examples/data-availability/detect-table-availability-issues.md +++ b/docs/examples/data-availability/detect-table-availability-issues.md @@ -98,7 +98,7 @@ The Sensor readouts category displays the values obtained by the sensors from th The Execution errors category displays any error that occurred during the check's execution. The actual value in this example is 0. -The check gives a valid result (notice the green square to the left of the check name). +The check gives a correct result (notice the green square to the left of the check name). ### **Synchronize the results with the cloud account** @@ -190,7 +190,7 @@ check run ``` Review the results which should be similar to the one below. -The number of failures is 0 and the check gives a valid result. +The number of failures is 0 and the check gives a correct result. ``` Check evaluation summary per table: diff --git a/docs/examples/data-completeness/detect-empty-or-incomplete-tables.md b/docs/examples/data-completeness/detect-empty-or-incomplete-tables.md index 89ded930a9..5cb708e5e4 100644 --- a/docs/examples/data-completeness/detect-empty-or-incomplete-tables.md +++ b/docs/examples/data-completeness/detect-empty-or-incomplete-tables.md @@ -96,7 +96,7 @@ The Execution errors category displays any error that occurred during the check's execution. The actual value of rows in this example is 18155, which is above the minimum threshold level set in the warning (1). -The check gives a valid result (notice the green square to the left of the check name). +The check gives a correct result (notice the green square to the left of the check name). Now you can be sure that you table is not empty. ### **Synchronize the results with the cloud account** @@ -279,7 +279,7 @@ check run ``` Review the results which should be similar to the one below. -The number of rows is above 1 and the check gives valid result. +The number of rows is above 1 and the check gives correct result. ``` Check evaluation summary per table: @@ -335,7 +335,7 @@ rows in a table does not fall below the minimum accepted count. If it does, you ## Next steps -- You haven't installed DQOps yet? Check the detailed guide on how to [install DQOps using pip](../../dqops-installation/install-dqops-using-pip.md) or [run DQO as a Docker container](../../dqops-installation/run-dqops-as-docker-container.md). +- You haven't installed DQOps yet? Check the detailed guide on how to [install DQOps using pip](../../dqops-installation/install-dqops-using-pip.md) or [run DQOps as a Docker container](../../dqops-installation/run-dqops-as-docker-container.md). - For details on the [row_count check used in this example, go to the check details section](../../checks/table/volume/row-count.md). - You might be interested in another completeness check that [evaluates that the number of nulls in a column does not exceed the maximum accepted count](../data-completeness/detect-null-values.md). - Would you like to add your own connection? Here you can find [information about supported databases and how to add new connection](../../data-sources/index.md). diff --git a/docs/examples/data-completeness/detect-null-values.md b/docs/examples/data-completeness/detect-null-values.md index 757a099b84..247bb78b50 100644 --- a/docs/examples/data-completeness/detect-null-values.md +++ b/docs/examples/data-completeness/detect-null-values.md @@ -30,7 +30,7 @@ In this example, we will set three maximum number thresholds levels for the chec - error: 10 - fatal: 15 -If you want to learn more about checks and threshold levels, please refer to the [DQO concept section](../../dqo-concepts/definition-of-data-quality-checks/index.md). +If you want to learn more about checks and threshold levels, please refer to the [DQOps concept section](../../dqo-concepts/definition-of-data-quality-checks/index.md). **VALUE** @@ -290,7 +290,7 @@ null values in a column does not exceed the minimum accepted count. If it does, ## Next steps -- You haven't installed DQOps yet? Check the detailed guide on how to [install DQOps using pip](../../dqops-installation/install-dqops-using-pip.md) or [run DQO as a Docker container](../../dqops-installation/run-dqops-as-docker-container.md). +- You haven't installed DQOps yet? Check the detailed guide on how to [install DQOps using pip](../../dqops-installation/install-dqops-using-pip.md) or [run DQOps as a Docker container](../../dqops-installation/run-dqops-as-docker-container.md). - For details on the [nulls_count check used in this example, go to the check details section](../../checks/column/nulls/nulls-count.md). - You might be interested in another completeness check that [evaluates that the number of rows in a table does not exceed the minimum accepted count](../data-completeness/detect-empty-or-incomplete-tables.md). - Would you like to add your own connection? Here you can find [information about supported databases and how to add new connection](../../data-sources/index.md). diff --git a/docs/examples/data-quality-monitoring/detect-empty-tables.md b/docs/examples/data-quality-monitoring/detect-empty-tables.md index 0bceefee8a..ce43371563 100644 --- a/docs/examples/data-quality-monitoring/detect-empty-tables.md +++ b/docs/examples/data-quality-monitoring/detect-empty-tables.md @@ -89,7 +89,7 @@ The Sensor readouts category displays the values obtained by the sensors from th The Execution errors category displays any error that occurred during the check's execution. The actual value in this example is 1 748 850, which is above the maximum threshold level set in the warning (1). -The check gives a valid result (notice the green square to the left of the check name). +The check gives a correct result (notice the green square to the left of the check name). ### **Synchronize the results with the cloud account** diff --git a/docs/examples/data-reasonability/percentage-of-false-values.md b/docs/examples/data-reasonability/percentage-of-false-values.md index 7671739869..03fc043c80 100644 --- a/docs/examples/data-reasonability/percentage-of-false-values.md +++ b/docs/examples/data-reasonability/percentage-of-false-values.md @@ -106,7 +106,7 @@ The Sensor readouts category displays the values obtained by the sensors from th The Execution errors category displays any error that occurred during the check's execution. The actual value in this example is 99%, which is above the minimum threshold level set in the warning (99%). -The check gives a valid result (notice the green square to the left of the check name). +The check gives a correct result (notice the green square to the left of the check name). ### **Synchronize the results with the cloud account** @@ -231,7 +231,7 @@ check run ``` Review the results which should be similar to the one below. -The check shows a valid results what means that the percentage of false values in the `invalidOcr` column exceeds 99%. +The check shows a correct results what means that the percentage of false values in the `invalidOcr` column exceeds 99%. ``` Check evaluation summary per table: diff --git a/docs/examples/data-uniqueness/percentage-of-duplicates.md b/docs/examples/data-uniqueness/percentage-of-duplicates.md index 1f48ac1923..4cc2c8f416 100644 --- a/docs/examples/data-uniqueness/percentage-of-duplicates.md +++ b/docs/examples/data-uniqueness/percentage-of-duplicates.md @@ -97,7 +97,7 @@ The Sensor readouts category displays the values obtained by the sensors from th The Execution errors category displays any error that occurred during the check's execution. The actual value in this example is 0%, which is below the maximum threshold level set in the warning (1.0%). -The check gives a valid result (notice the green square to the left of the check name). +The check gives a correct result (notice the green square to the left of the check name). ### **Synchronize the results with the cloud account** @@ -219,7 +219,7 @@ check run ``` Review the results which should be similar to the one below. -The percent of the duplicate values in the `unique_key` column is below 5.0% and the check gives valid result. +The percent of the duplicate values in the `unique_key` column is below 5.0% and the check gives correct result. ``` Check evaluation summary per table: diff --git a/docs/examples/data-validity/percentage-of-texts-matching-date-regex.md b/docs/examples/data-validity/percentage-of-texts-matching-date-regex.md index 43a86b117e..c0eeedf322 100644 --- a/docs/examples/data-validity/percentage-of-texts-matching-date-regex.md +++ b/docs/examples/data-validity/percentage-of-texts-matching-date-regex.md @@ -22,20 +22,20 @@ The `source_date` column contains non-standard date format. We want to verify th **SOLUTION** We will verify the data of `bigquery-public-data.america_health_rankings.ahr` using monitoring -[text_matching_date_pattern_percent](../../checks/column/patterns/text-matching-date-pattern-percent.md) column check. -Our goal is to verify if the percentage of values matches the indicated by the user date format on `source_date` column does not fall below the set thresholds. +[text_not_matching_date_pattern_percent](../../checks/column/patterns/text-not-matching-date-pattern-percent.md) column check. +Our goal is to verify if the percentage of invalid values that are not matching the user expected date format on the `source_date` column does not exceed the set thresholds. In this example, we will set three minimum percentage thresholds levels for the check: -- warning: 99.0% -- error: 98.0% -- fatal: 95.0% +- warning: 1.0% +- error: 2.0% +- fatal: 5.0% If you want to learn more about checks and threshold levels, please refer to the [DQOps concept section](../../dqo-concepts/definition-of-data-quality-checks/index.md). **VALUE** -If the percentage of data falls below 99.0%, a warning alert will be triggered. +If the percentage of data exceed 1.0%, a warning alert will be triggered. ## Data structure @@ -211,15 +211,15 @@ spec: monitoring_checks: daily: patterns: - daily_text_matching_date_pattern_percent: + daily_text_not_matching_date_pattern_percent: parameters: date_format: YYYY-MM-DD warning: - min_percent: 99.0 + max_percent: 1.0 error: - min_percent: 98.0 + max_percent: 2.0 fatal: - min_percent: 95.0 + max_percent: 5.0 ``` ## Run the checks in the example using the DQOps Shell @@ -262,45 +262,41 @@ SELECT WHEN COUNT(analyzed_table.`source_date`) = 0 THEN NULL ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`source_date`) IS NOT NULL + WHEN SAFE.PARSE_DATE('%Y-%m-%d', analyzed_table.`source_date`) IS NULL THEN 1 ELSE 0 END - ) / COUNT(*) - END AS actual_value, - CURRENT_TIMESTAMP() AS time_period, - TIMESTAMP(CURRENT_TIMESTAMP()) AS time_period_utc + ) / COUNT(analyzed_table.`source_date`) + END AS actual_value FROM `bigquery-public-data`.`america_health_rankings`.`ahr` AS analyzed_table -GROUP BY time_period, time_period_utc -ORDER BY time_period, time_period_utc ************************************************** ``` -You can also see the results returned by the sensor. The actual value in this example is 0.0%, which is below the minimum -threshold level set in the Fatal error (95.0%). +You can also see the results returned by the sensor. The actual value in this example is 100.0%, which is above the minimum +threshold level set in the Fatal error (5.0% of invalid records). ``` ************************************************** -Finished executing a sensor for a check text_matching_date_pattern_percent on the table america_health_rankings.ahr using a sensor definition column/patterns/text_matching_date_pattern_percent, sensor result count: 1 +Finished executing a sensor for a check text_not_matching_date_pattern_percent on the table america_health_rankings.ahr using a sensor definition column/patterns/text_not_matching_date_pattern_percent, sensor result count: 1 Results returned by the sensor: -+------------+------------------------+------------------------+ -|actual_value|time_period |time_period_utc | -+------------+------------------------+------------------------+ -|0.0 |2023-04-26T11:01:24.538Z|2023-04-26T11:01:24.538Z| -+------------+------------------------+------------------------+ ++------------+ +|actual_value| ++------------+ +|100.0 | ++------------+ ************************************************** ``` In this example, we have demonstrated how to use DQOps to verify the validity of data in a column. -By using the [text_matching_date_pattern_percent](../../checks/column/patterns/text-matching-date-pattern-percent.md) column check, we can monitor that +By using the [text_not_matching_date_pattern_percent](../../checks/column/patterns/text-not-matching-date-pattern-percent.md) column check, we can monitor that the percentage of strings matching the date format regex in a column does not exceed the maximum accepted percentage. If it does, you will get a warning, error or fatal results. ## Next steps - You haven't installed DQOps yet? Check the detailed guide on how to [install DQOps using pip](../../dqops-installation/install-dqops-using-pip.md) or [run DQOps as a Docker container](../../dqops-installation/run-dqops-as-docker-container.md). -- For details on the [text_matching_date_pattern_percent check used in this example, go to the check details section](../../checks/column/patterns/text-matching-date-pattern-percent.md). +- For details on the [text_not_matching_date_pattern_percent check used in this example, go to the check details section](../../checks/column/patterns/text-not-matching-date-pattern-percent.md). - You might be interested in another validity check that [evaluates that the percentage of valid currency code strings in the monitored column does not fall below set thresholds.](./percentage-of-valid-currency-codes.md). - With DQOps, you can easily customize when the checks are run at the level of the entire connection, table, or individual check. [Learn more about how to set schedules here](../../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md). - DQOps allows you to keep track of the issues that arise during data quality monitoring and send alert notifications directly to Slack. Learn more about [incidents](../../working-with-dqo/managing-data-quality-incidents-with-dqops.md) and [Slack notifications](../../integrations/slack/configuring-slack-notifications.md). diff --git a/docs/examples/data-validity/percentage-of-valid-latitude-and-longitude.md b/docs/examples/data-validity/percentage-of-valid-latitude-and-longitude.md index 65a12517a1..20f577001f 100644 --- a/docs/examples/data-validity/percentage-of-valid-latitude-and-longitude.md +++ b/docs/examples/data-validity/percentage-of-valid-latitude-and-longitude.md @@ -106,7 +106,7 @@ The Execution errors category displays any error that occurred during the check' The actual value in this example is 99.24% for both longitude and latitude checks, which is above the minimum threshold level set in the warning (99.0%). -The check gives a valid result (notice the green square to the left of the check name). +The check gives a correct result (notice the green square to the left of the check name). ### **Synchronize the results with the cloud account** diff --git a/docs/examples/data-validity/percentage-of-valid-uuid.md b/docs/examples/data-validity/percentage-of-valid-uuid.md index 0333cf5133..7f301ca516 100644 --- a/docs/examples/data-validity/percentage-of-valid-uuid.md +++ b/docs/examples/data-validity/percentage-of-valid-uuid.md @@ -16,20 +16,20 @@ We want to verify the percent of valid UUID on `uuid` column does not fall below **SOLUTION** -We will verify the data using monitoring [valid_uuid_format_percent](../../checks/column/patterns/valid-uuid-format-percent.md) column check. -Our goal is to verify that the percent of valid UUID values in a `uuid` column does not fall below the set threshold. +We will verify the data using monitoring [invalid_uuid_format_percent](../../checks/column/patterns/invalid-uuid-format-percent.md) column check. +Our goal is to verify that the percent of invalid UUID values in a `uuid` column does not exceed the set threshold. In this example, we will set three minimum percent thresholds levels for the check: -- warning: 99 -- error: 98 -- fatal: 95 +- warning: 1% +- error: 2% +- fatal: 5% If you want to learn more about checks and threshold levels, please refer to the [DQOps concept section](../../dqo-concepts/definition-of-data-quality-checks/index.md). **VALUE** -If the percentage of valid UUID values fall below 99%, a warning alert will be triggered. +If the percentage of invalid UUID values exceeds 1%, a warning alert will be triggered. ## Data structure @@ -143,11 +143,11 @@ This dashboard allows filtering data by: The YAML configuration file stores both the table details and checks configurations. -In this example, we have set three minimum percent thresholds levels for the check: +In this example, we have set three maximum percent thresholds levels for the check: -- warning: 99 -- error: 98 -- fatal: 95 +- warning: 1 +- error: 2 +- fatal: 5 The highlighted fragments in the YAML file below represent the segment where the monitoring `daily_valid_uuid_format_percent` check is configured. @@ -168,13 +168,13 @@ spec: monitoring_checks: daily: patterns: - daily_valid_uuid_format_percent: + daily_invalid_uuid_format_percent: warning: - min_percent: 99.0 + max_percent: 1.0 error: - min_percent: 98.0 + maz_percent: 2.0 fatal: - min_percent: 95.0 + max_percent: 5.0 result: type_snapshot: column_type: INT64 @@ -192,7 +192,7 @@ check run ``` Review the results which should be similar to the one below. -The percent of valid UUID values in the `uuid` column is below 95 and the check raised fatal error. +The percent of valid UUID values in the `uuid` column is above 5% and the check raised fatal error. ``` Check evaluation summary per table: @@ -218,14 +218,14 @@ Executing SQL on connection valid_uuid_percent (bigquery) SQL to be executed on the connection: SELECT CASE - WHEN COUNT(*) = 0 THEN 100.0 + WHEN COUNT(analyzed_table.`uuid`) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`uuid` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") + WHEN NOT REGEXP_CONTAINS(SAFE_CAST(analyzed_table.`uuid` AS STRING), r"^[0-9a-fA-F]{8}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{4}[\s-]?[0-9a-fA-F]{12}$") THEN 1 ELSE 0 END - ) / COUNT(*) + ) / COUNT(analyzed_table.`uuid`) END AS actual_value, CURRENT_TIMESTAMP() AS time_period, TIMESTAMP(CURRENT_TIMESTAMP()) AS time_period_utc @@ -235,30 +235,30 @@ ORDER BY time_period, time_period_utc ************************************************** ``` -You can also see the results returned by the sensor. The actual value in this example is 75%, which is below the minimum -threshold level set in the warning (99%). +You can also see the results returned by the sensor. The actual value in this example is 25%, which is above the maximum +threshold level set in the warning (1%). ``` ************************************************** -Finished executing a sensor for a check valid_uuid_format_percent on the table dqo_ai_test_data.uuid_test_7014625119307825231 using a sensor definition column/patterns/valid_uuid_format_percent, sensor result count: 1 +Finished executing a sensor for a check invalid_uuid_format_percent on the table dqo_ai_test_data.uuid_test_7014625119307825231 using a sensor definition column/patterns/invalid_uuid_format_percent, sensor result count: 1 Results returned by the sensor: +------------+------------------------+------------------------+ |actual_value|time_period |time_period_utc | +------------+------------------------+------------------------+ -|75.0 |2023-05-22T12:08:24.199Z|2023-05-22T12:08:24.199Z| +|25.0 |2023-05-22T12:08:24.199Z|2023-05-22T12:08:24.199Z| +------------+------------------------+------------------------+ ************************************************** ``` In this example, we have demonstrated how to use DQOps to verify the validity of data in a column. -By using the [valid_uuid_format_percent](../../checks/column/patterns/valid-uuid-format-percent.md) column check, we can monitor that -the percentage of valid UUID values in a column does not fall below the minimum accepted percentage. If it does, you will get a warning, error or fatal results. +By using the [invalid_uuid_format_percent](../../checks/column/patterns/invalid-uuid-format-percent.md) column check, we can monitor that +the percentage of valid UUID values in a column does exceed the maximum accepted percentage. If it does, you will get a warning, error or fatal results. ## Next steps - You haven't installed DQOps yet? Check the detailed guide on how to [install DQOps using pip](../../dqops-installation/install-dqops-using-pip.md) or [run DQOps as a Docker container](../../dqops-installation/run-dqops-as-docker-container.md). -- For details on the [valid_uuid_format_percent check used in this example, go to the check details section](../../checks/column/patterns/valid-uuid-format-percent.md). +- For details on the [valid_uuid_format_percent check used in this example, go to the check details section](../../checks/column/patterns/invalid-uuid-format-percent.md). - You might be interested in another validity check that [evaluates that ensures that the percentage of rows containing valid currency codes does not exceed set thresholds](./percentage-of-values-that-contains-usa-zipcode.md). - With DQOps, you can easily customize when the checks are run at the level of the entire connection, table, or individual check. [Learn more about how to set schedules here](../../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md). - DQOps allows you to keep track of the issues that arise during data quality monitoring and send alert notifications directly to Slack. Learn more about [incidents](../../working-with-dqo/managing-data-quality-incidents-with-dqops.md) and [Slack notifications](../../integrations/slack/configuring-slack-notifications.md). \ No newline at end of file diff --git a/docs/examples/index.md b/docs/examples/index.md index 6da8dc1dd7..7059ca994b 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -56,10 +56,10 @@ These examples use openly available datasets from [Google Cloud](https://cloud.g | [Detect invalid IP4 address](./data-validity/detect-invalid-ip4-addresses.md) | This example shows how to detect that the number of invalid IP4 address in a column does not exceed a set threshold using [invalid_ip4_address_format_found](../checks/column/patterns/invalid-ip4-address-format-found.md) check. | DQOps dataset | | [Percentage of negative values](./data-validity/percentage-of-negative-values.md) | This example shows how to detect that the percentage of negative values in a column does not exceed a set threshold using [negative_values_percent](../checks/column/numeric/negative-values-percent.md) check. | [Link](https://www.worldometers.info/world-population/population-by-country/) | | [Percentage of rows passing SQL condition](./data-validity/percentage-of-rows-passing-sql-condition.md) | This example shows how to detect that the percentage of passed sql condition in a column does not fall below a set threshold using [sql_condition_passed_percent](../checks/table/custom_sql/sql-condition-passed-percent-on-table.md) check. | [Link](https://www.americashealthrankings.org/about/methodology/our-reports) | -| [Percentage of texts matching a date pattern](./data-validity/percentage-of-texts-matching-date-regex.md) | This example shows how to detect that the percentage of texts matching the date format regex in a column does not exceed a set threshold using [text_matching_date_pattern_percent](../checks/column/patterns/text-matching-date-pattern-percent.md) check. | [Link](https://www.americashealthrankings.org/about/methodology/our-reports) | +| [Percentage of texts not matching a date pattern](./data-validity/percentage-of-texts-matching-date-regex.md) | This example shows how to detect that the percentage of texts matching the date format regex in a column does not exceed a set threshold using [text_not_matching_date_pattern_percent](../checks/column/patterns/text-not-matching-date-pattern-percent.md) check. | [Link](https://www.americashealthrankings.org/about/methodology/our-reports) | | [Percentage of valid currency codes](./data-validity/percentage-of-valid-currency-codes.md) | This example shows how to detect that the percentage of valid currency codes in a column does not fall below a set threshold using [text_valid_currency_code_percent](../checks/column/accepted_values/text-valid-currency-code-percent.md) check. | DQOps dataset | | [Percentage of valid latitude and longitude](./data-validity/percentage-of-valid-latitude-and-longitude.md) | This example shows how to detect that the percentage of valid latitude and longitude values remain above a set threshold using [valid_latitude_percent](../checks/column/numeric/valid-latitude-percent.md) and [valid_longitude_percent](../checks/column/numeric/valid-longitude-percent.md)checks. | [Link](https://data.austintexas.gov/Utilities-and-City-Services/Austin-311-Public-Data/xwdj-i9he) | -| [Percentage of valid UUID](./data-validity/percentage-of-valid-uuid.md) | This example shows how to detect that the percentage of valid UUID values in a column does not fall below a set threshold using [valid_uuid_format_percent](../checks/column/patterns/valid-uuid-format-percent.md) check. | DQOps dataset | +| [Percentage of invalid UUID](./data-validity/percentage-of-valid-uuid.md) | This example shows how to detect that the percentage of valid UUID values in a column does not fall below a set threshold using [invalid_uuid_format_percent](../checks/column/patterns/invalid-uuid-format-percent.md) check. | DQOps dataset | | [Percentage of rows containing USA zip codes](./data-validity/percentage-of-values-that-contains-usa-zipcode.md) | This example shows how to detect USA zip codes in text columns by measuring the percentage of rows containing a zip code using the [contains_usa_zipcode_percent](../checks/column/pii/contains-usa-zipcode-percent.md) check. | [Link](https://data.austintexas.gov/Utilities-and-City-Services/Austin-311-Public-Data/xwdj-i9he) | ### **Schema** @@ -188,7 +188,7 @@ Run the activated check using the **Run check** button. The results of the run check are shown as a color square -- Green for a valid result +- Green for a correct result - Yellow for a warning - Orange for an error - Red for a fatal error diff --git a/docs/examples/schema/detect-table-schema-changes.md b/docs/examples/schema/detect-table-schema-changes.md index 969e30d852..9c680282cd 100644 --- a/docs/examples/schema/detect-table-schema-changes.md +++ b/docs/examples/schema/detect-table-schema-changes.md @@ -101,7 +101,7 @@ The Check results category shows the severity level that result from the verific The Sensor readouts category displays the values obtained by the sensors from the data source. The Execution errors category displays any error that occurred during the check's execution. -All the checks in the example gives valid results (notice the green square to the left of the checks name). +All the checks in the example gives correct results (notice the green square to the left of the checks name). ### **Modify the table schema** diff --git a/docs/getting-started/add-data-source-connection.md b/docs/getting-started/add-data-source-connection.md index 900dbce077..afbfafc9ab 100644 --- a/docs/getting-started/add-data-source-connection.md +++ b/docs/getting-started/add-data-source-connection.md @@ -1,10 +1,10 @@ --- -title: Creating a connection to a data source +title: Creating a connection to a data source and starting data quality assessment --- -# Creating a connection to a data source -This guide shows how to connect a data source to DQOps, import the metadata, and operate with the DQOps user interface. +# Creating a connection to a data source and profiling data +This guide shows how to connect a data source to DQOps, import the metadata, and start data quality assessment. -## Overview +## Creating a connection to a data source After [installation and starting DQOps](installation.md), we describe how to import locally stored CSV file using the user interface, initiate automatic monitoring and run basic statistics. @@ -19,62 +19,72 @@ You can find more information about [navigating the DQOps user interface here](. Instead of importing a CSV file, you can follow the manuals for connecting to other data sources and continue this getting started guide from the [import metadata using the user interface](#import-metadata-using-the-user-interface) step. + DQOps supports integration with relational databases, data lakes, cloud data warehouses, object storage services, + data processing frameworks, query engines and flat files. You can find [the full list of supported datasources here](../data-sources/index.md). + Links to some supported data sources are shown below. [![Athena](https://dqops.com/docs/images/connections/athena2.png){ class=glightbox-ignored-image }](../data-sources/athena.md) -       [![PostgreSQL](https://dqops.com/docs/images/connections/postgresql.png){ class=glightbox-ignored-image }](../data-sources/postgresql.md) +       [![Redshift](https://dqops.com/docs/images/connections/amazon-redshift1.png){ class=glightbox-ignored-image }](../data-sources/redshift.md)       [![BigQuery](https://dqops.com/docs/images/connections/bigquery.png){ class=glightbox-ignored-image }](../data-sources/bigquery.md) +       [![Snowflake](https://dqops.com/docs/images/connections/snowflake.png){ class=glightbox-ignored-image }](../data-sources/snowflake.md) +       [![PostgreSQL](https://dqops.com/docs/images/connections/postgresql.png){ class=glightbox-ignored-image }](../data-sources/postgresql.md) +       [![Oracle](https://dqops.com/docs/images/connections/oracle2.png){ class=glightbox-ignored-image }](../data-sources/oracle.md) +       [![MySQL](https://dqops.com/docs/images/connections/mysql.png){ class=glightbox-ignored-image }](../data-sources/mysql.md) +       [![SQL Server](https://dqops.com/docs/images/connections/microsoft-sql-server.png){ class=glightbox-ignored-image }](../data-sources/sql-server.md) +       [![Spark](https://dqops.com/docs/images/connections/spark.png){ class=glightbox-ignored-image }](../data-sources/spark.md) +       [![Databricks](https://dqops.com/docs/images/connections/databricks.png){ class=glightbox-ignored-image }](../data-sources/databricks.md)       [![Parquet](https://dqops.com/docs/images/connections/parquet-icon2.png){ class=glightbox-ignored-image }](../data-sources/parquet.md) ### Download the sample CSV file -In the example, we will use a sample CSV file named **austin_crime.csv**. This file contains a fragment of the [public dataset Austin Crime Data](https://console.cloud.google.com/marketplace/details/city-of-austin/austin-crime), +In the example, we will use a sample CSV file named **orders.csv**. This file contains a fragment of the [public dataset of a fictitious ecommerce clothing site TheLook](https://console.cloud.google.com/marketplace/product/bigquery-public-data/thelook-ecommerce), but feel free to use your own CSV file if you prefer. -To download the sample CSV file, [open the GitHub webpage](https://github.com/dqops/dqo/blob/develop/dqops/sampledata/files/csv/austin_crime_sample/austin_crime.csv), +To download the sample CSV file, [open the GitHub webpage](https://github.com/dqops/dqo/blob/develop/dqops/sampledata/files/csv/thelook-ecommerce/orders.csv), click on the three dots button in the upper right corner, and select the **Download** command. -![Download sample CSV file from GitHub](https://dqops.com/docs/images/getting-started/github-download.png) +![Download sample CSV file from GitHub](https://dqops.com/docs/images/getting-started/github-download.png){ loading=lazy; width="400px" } In our example, we have downloaded the file to a folder named __demo_files__. Remember the absolute path to the file, as we will need it when configuring the connection. -In our example, the path is **C:\demo_files\austin_crime.csv** +In our example, the path is **C:\demo_files\orders.csv** -![Adding connection](https://dqops.com/docs/images/getting-started/file-explorer.png) +![Adding connection](https://dqops.com/docs/images/getting-started/file-explorer3.png){ loading=lazy; width="700px" } Below is a table that shows a fragment of the data included in the sample CSV file. -| unique_key | address | census_tract | clearance_date | clearance_status | council_district_code | description | district | latitude | longitude | location | location_description | primary_type | timestamp | x_coordinate | y_coordinate | year | zipcode | -|------------|--------------------------------|--------------|--------------------------------|------------------|-----------------------|--------------------------|----------|----------|-----------|----------|----------------------|--------------|--------------------------------|--------------|--------------|------|---------| -| 2015821204 | "1713 MULLEN DR Austin, TX" | | 2015-03-25 12:00:00.000000 UTC | Not cleared | | THEFT | UK | | | | 1713 MULLEN DR | Theft | 2015-03-23 12:00:00.000000 UTC | | | 2015 | | -| 2015150483 | "Austin, TX" | | 2015-01-27 12:00:00.000000 UTC | Not cleared | | RAPE | B | | | | nan | Rape | 2015-01-15 12:00:00.000000 UTC | | | 2015 | | -| 2015331540 | "5510 S IH 35 SVRD Austin, TX" | | 2015-02-11 12:00:00.000000 UTC | Not cleared | | BURGLARY OF VEHICLE | UK | | | | 5510 S IH 35 SVRD | Theft | 2015-02-02 12:00:00.000000 UTC | | | 2015 | | -| 2015331238 | "7928 US HWY 71 W Austin, TX" | | 2015-02-12 12:00:00.000000 UTC | Not cleared | | THEFT OF HEAVY EQUIPMENT | UK | | | | 7928 US HWY 71 W | Theft | 2015-02-02 12:00:00.000000 UTC | | | 2015 | | -| ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | +| order_id | user_id | status | gender | created_at | returned_at | shipped_at | delivered_at | num_of_item | +|----------|---------|------------|--------|--------------------|-------------|--------------------|-------------------|-------------| +| 1 | 3 | Shipped | F | 8/12/2019 15:15:00 | | 8/15/2019 14:08:00 | | 1 | +| 2 | 5 | Shipped | M | 1/20/2022 9:17:00 | | 1/22/2022 2:25:00 | | 1 | +| 3 | 6 | Processing | F | 7/23/2022 11:33:00 | | | | 4 | +| 4 | 6 | Completed | F | 3/1/2020 11:33:00 | | 3/2/2020 15:57:00 | 3/4/2020 16:32:00 | 3 | +| ... | ... | ... | ... | ... | ... | ... | ... | ... | -## Add connection to a CSV file using the user interface +## Add a connection to a CSV file using the user interface ### **Navigate to the connection settings** To navigate to the CSV connection settings: -1. Go to the **Data Sources** section and click **+ Add connection** button in the upper left corner. +1. Go to the **Data Sources** section and click the **+ Add connection** button in the upper left corner. - ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection.png) + ![Adding connection](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection2.png){ loading=lazy; width="1200px" } 2. Select **CSV** connection type. - ![Selecting CSV database type](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection-csv.png){ loading=lazy; width="1200px" } + ![Selecting CSV database type](https://dqops.com/docs/images/working-with-dqo/adding-connections/adding-connection-csv2.png){ loading=lazy; width="1200px" } ### **Fill in the connection settings** -After navigating to the CSV connection settings, you will need to fill in the connection details. +After opening the CSV connection settings, fill in the connection details. -To import a CSV file stored locally, we only need to fill in settings described below. -You can read about other settings and how to import CSV files from AWS S3 in the [CSV connection documentation](../data-sources/csv.md). +If you are importing a CSV file stored locally, you only need to complete the settings described below. +For information on other settings and how to import CSV files from AWS S3, refer to [the CSV connection documentation](../data-sources/csv.md)." ![Adding connection](https://dqops.com/docs/images/getting-started/connection-settings-csv-filled1.png) @@ -84,91 +94,135 @@ You can read about other settings and how to import CSV files from AWS S3 in the | Virtual schema name | An alias for the parent directory with data. We leave the default value "files". | | Path | The path prefix to the parent directory with data. The path must be absolute. || Path | The path prefix to the parent directory with data. The path must be absolute. The virtual schema name is a value of the directories mapping. | -After filling in the connection settings, click the **Test Connection** button to test the connection. +After entering the connection settings, click on the **Test Connection** button to test the connection. It will inform you if the path to the CSV file is correct. -Click the **Save** button when the test is successful to add a new connection. -Otherwise, you can check the details of what went wrong. +If the test is successful, click the **Save** button to add a new connection. If the test fails, you can review the details to identify the issue. ## **Import metadata using the user interface** -When you add a new connection, it will appear in the tree view on the left, and you will be redirected to the **Import metadata** screen, +When you add a new connection, it will appear in the tree view on the left. You will then be redirected to the **Import metadata** screen, where you can import files. 1. Import the "files" schema by clicking the **Import tables** button. - ![Importing schemas](https://dqops.com/docs/images/getting-started/importing-schema-files.png){ loading=lazy; width="1200px" } + ![Importing schemas](https://dqops.com/docs/images/getting-started/importing-schema-files2.png){ loading=lazy; width="1200px" } + +2. Import the "orders.csv" file by clicking on the **Import all tables** button in the upper right corner. + + ![Importing tables](https://dqops.com/docs/images/getting-started/importing-orders-csv4.png){ loading=lazy; width="1200px" } -2. Click on the checkbox next to the file name **austin_crime.csv** and import it by clicking **Import selected tables** button in the upper right corner. +DQOps will create a table from the **orders.csv** file in the virtual schema "files", which will allow you to profile it and monitor its data quality. - ![Importing tables](https://dqops.com/docs/images/getting-started/importing-tables-austin-crime-csv.png){ loading=lazy; width="1200px" } -DQOps will create a table from the **austin_crime.csv** file in the virtual schema "files", which will allow you to profile it and monitor its data quality. +## Start data quality assessment -## Initiate automatic monitoring and review scheduling +DQOps simplifies the data quality assessment process which is a routine activity for data governance and quality teams. -Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). -These checks include row count, table availability, and checks detecting schema changes. They are scheduled to run daily at 12:00 p.m. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -[**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess -the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with -various types of checks and determining the most suitable ones for regular data quality monitoring. +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) -[**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are -standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. -These checks capture a single data quality result for the entire table or column. There are two categories -of monitoring checks: *daily* checks and *monthly* checks. +!!! info "Automatically activated checks" -Upon import, you will receive information at the top of the page, it is called an **Advisor**. -If you click on the orange bar, you can open the Advisor. Within the Advisor, you can collect basic statistics, -run profiling checks, or modify the schedule for newly imported tables. + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. -### Run basic statistics and profiling checks with the Advisor +### Default data profiling checks -To Run basic statistics and profiling checks, click on the appropriate buttons on the advisor. +Below is a list of the [data profiling checks](../dqo-concepts/data-observability.md) automatically activated according +to default Table and Column-level data quality policies. -We will evaluate the results from basic statistics and profiling checks at the next step of the Getting started. +| Target | Check name | Description | +|--------|------------------------------------------------------------------------------------------|------------------------------------------------| +| column | [profile contains usa phone percent](../checks/column/pii/contains-usa-phone-percent.md) | Detects USA phone numbers inside text columns. | +| column | [profile contains email percent](../checks/column/pii/contains-email-percent.md) | Detects emails inside text columns | -![Running basic statistics and profiling checks](https://dqops.com/docs/images/getting-started/running-basics-statistics-and-profiling-checks-csv.png){ loading=lazy; width="1200px" } +### Default daily monitoring checks +Below is a list of the [daily monitoring checks](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) +automatically activated according to default Table and Column-level data quality policies. -### Review scheduling with the Advisor +| Target | Check name | Description | +|--------|-----------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| table | [daily row count](../checks/table/volume/row-count.md) | Captures the row count of the table every day and identifies empty tables. | +| table | [daily row count anomaly](../checks/table/volume/row-count-anomaly.md) | Detects day-to-day anomalies in the table volume. Raises a data quality issue when the increase or decrease in the row count is in the top 1% of most significant changes. | +| table | [daily row count change](../checks/table/volume/row-count-change.md) | Detects significant day-to-day changes in the table volume. Raises a data quality issue when the increase or decrease in the row count is greater than 10%. | +| table | [daily table availability](../checks/table/availability/table-availability.md) | Verifies that a table exists, can be accessed, and queried without errors. Detects corrupted tables and expired credentials to data sources. | +| table | [daily column count](../checks/table/schema/column-count.md) | Retrieves the metadata of the monitored table from the data source and counts the number of columns. | +| table | [daily column count changed](../checks/table/schema/column-count-changed.md) | Detects whether the number of columns in a table has changed since the last time the check (checkpoint) was run. | +| column | [daily nulls count](../checks/column/nulls/nulls-count.md) | Counts null values in every column without raising any data quality issues. | +| column | [daily nulls percent](../checks/column/nulls/nulls-percent.md) | Measures the percentage of null values in every column without raising any data quality issues. | +| column | [daily nulls percent anomaly](../checks/column/nulls/nulls-percent-anomaly.md) | Measures the percentage of null values in every column and detects anomalous changes in the percentage of null value. Raises a data quality issue for the top 1% biggest day-to-day changes. | +| column | [daily not nulls count](../checks/column/nulls/not-nulls-count.md) | Verifies that a column contains a minimum number of non-null values | +| column | [daily not nulls percent](../checks/column/nulls/not-nulls-percent.md) | Measures the percentage of null values in every column without raising any data quality issues. | +| column | [daily empty column found](../checks/column/nulls/empty-column-found.md) | Detects empty columns that contain only null values. | +| column | [daily distinct count anomaly](../checks/column/uniqueness/distinct-count-anomaly.md) | Counts distinct values in every column and detects anomalous changes in the percentage of null value. Raises a data quality issue for the top 1% biggest day-to-day changes of the count of distinct values. | +| column | [daily detected datatype in text changed](../checks/column/datatype/detected-datatype-in-text-changed.md) | Scans all values in a text column and detects the data type of all values in a column. Raises a data quality issue when the type of texts changes. For example, when a column contained always numeric values, but a text value was found. | +| column | [daily column exists](../checks/column/schema/column-exists.md) | Reads the metadata of the monitored table and verifies that the column still exists in the data source. | +| column | [daily column type changed](../checks/column/schema/column-type-changed.md) | Detects if the data type of the column has changed since the last time it was retrieved. | -To review scheduling for profiling and daily monitoring checks, click on the **Review scheduling** button. +For more [general information about checks, see the DQOps concepts](../dqo-concepts/definition-of-data-quality-checks/index.md) section. -![Review scheduling](https://dqops.com/docs/images/getting-started/review-scheduling-csv.png){ loading=lazy; width="1200px" } -You will be linked to **Data Source** section, **Schedule** tab where you can review scheduling settings for the added connection. +### Review scheduling -The scheduling is enabled by default. You can turn it off by clicking the notification icon in the upper right corner and +To review scheduling for profiling and daily monitoring checks + +1. Go to the **Data source** section. +2. Select the connection from the tree view on the left. +3. Select the **Schedule** tab where you can review scheduling settings for the added connection. + +![Review scheduling](https://dqops.com/docs/images/getting-started/review-scheduling-csv4.png){ loading=lazy; width="1200px" } + +The scheduling is enabled by default. You can turn it off by clicking the **Notification icon** in the upper right corner and then clicking the **Job scheduler** toggle button. -![Reviewing data source details](https://dqops.com/docs/images/getting-started/reviewing-data-source-section-csv1.png){ loading=lazy; width="1200px" } +![Reviewing data source details](https://dqops.com/docs/images/getting-started/reviewing-job-scheduler2.png){ loading=lazy; width="1200px" } ## Explore the connection-level tabs in the Data sources section -There are several tabs to explore in the **Data sources** section that differs depending on the selection of the elements in the tree view on the left (connection, schema, table or column): +There are several tabs to explore in the **Data sources** section that differ depending on the selection of the elements in the tree view on the left (connection, schema, table or column): The following tabs are shown at the connection level: +- **Connection:** Provides details about the connection parameters. +- **Schedule:** Allows you to [configure of the check execution schedule](../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md) at the connection level. +- **Comments:** Allows adding comments to your connection. +- **Labels:** Allows adding labels to your connection. +- **Schemas:** Allow importing schemas and tables. +- **Data quality summary:** Displays summaries of the data quality status. You have the option to choose one of two subtabs: + - **Tables:** This subtab provides a summary of the data quality status for tables in this connection. + - **Columns:** This subtab provides a summary of the data quality status for columns in this connection. +- **Default grouping template:** Allows setting up data grouping globally at the data source level. [Learn how to configure data grouping](../working-with-dqo/set-up-data-grouping-for-data-quality-checks.md). +- **Incident grouping**: Allows configuring incidents grouping level. [Learn more about incidents](../dqo-concepts/grouping-data-quality-issues-to-incidents.md) that let you keep track of the issues that arise during data quality monitoring. +- **Notifications:** Allows configuring incidents and Webhooks for notifications. [Learn how to configure notifications](../dqo-concepts/grouping-data-quality-issues-to-incidents.md#configure-notification-for-an-incident) whenever a new incident is created or modified. ## Explore the table-level tabs in the Data sources section At the table level in the **Data sources** section, there are the following tabs: -- **Table** - provide details about the table and allows you to add filter, priority or stage name (for example, "Ingestion"). -- **Schedule** - allows setting schedule for running checks. [Learn how to configure schedules](../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md). -- **Comments** - allows adding comments to your tables. -- **Labels** - allows adding labels to your tables. -- **Data groupings** - allows setting up data grouping at the table level. [Learn how to configure data grouping](../working-with-dqo/set-up-data-grouping-for-data-quality-checks.md). -- **Date and time columns** - allows [configuring event and ingestion timestamp columns for timeliness checks](../working-with-dqo/run-data-quality-checks.md#configure-event-and-ingestion-timestamp-columns-for-timeliness-checks), as well as [date or datetime column for partition checks](../working-with-dqo/run-data-quality-checks.md#configure-date-or-datetime-column-for-partition-checks). -- **Incident configuration** - allows configuring incidents. [Learn more about incidents](../working-with-dqo/managing-data-quality-incidents-with-dqops.md) that let you keep track of the issues that arise during data quality monitoring. - -You can check the details of the imported table by expanding the tree view on the left and selecting the "austin_crime.csv" table. +- **Table:** Provides details about the table and allows you to add filter, priority or stage name (for example, "Ingestion"). +- **Schedule:** Allows setting schedule for running checks. [Learn how to configure schedules](../working-with-dqo/configure-scheduling-of-data-quality-checks/index.md). +- **Comments:** Allows adding comments to your tables. +- **Labels:** Allows adding labels to your tables. +- **Data groupings:** Allows setting up data grouping at the table level. [Learn how to configure data grouping](../working-with-dqo/set-up-data-grouping-for-data-quality-checks.md). +- **Date and time columns:** Allows [configuring event and ingestion timestamp columns for timeliness checks](../categories-of-data-quality-checks/how-to-detect-timeliness-and-freshness-issues.md#configure-dqops-for-timeliness-checks), as well as [date or datetime column for partition checks](../dqo-concepts/definition-of-data-quality-checks/partition-checks.md#setting-up-date-partitioning-column). +- **Incident configuration:** Allows configuring incidents. [Learn more about incidents](../working-with-dqo/managing-data-quality-incidents-with-dqops.md) that let you keep track of the issues that arise during data quality monitoring. -![Reviewing table details](https://dqops.com/docs/images/getting-started/reviewing-table-details-csv.png){ loading=lazy; width="1200px" } +![Reviewing table details](https://dqops.com/docs/images/getting-started/reviewing-table-level-tabs2.png){ loading=lazy; width="1200px" } ## Next step -Now that you have connected a data source and initiated automatic monitoring, -it is time to [review the results and run additional checks](review-results-and-run-monitoring-checks.md). \ No newline at end of file +Now that you have connected a data source and initiated data assessment, it is time to +[review the results, automatically configure data quality checks to detect the most common data quality issues and create an incident notification](review-results-and-run-monitoring-checks.md). \ No newline at end of file diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 877370f61a..db03321a23 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -2,13 +2,14 @@ title: Getting started with DQOps Data Quality Operations Center --- # Getting started with DQOps Data Quality Operations Center -This guide provides a quick tutorial on how to get started with DQOps using the web interface. You will add a data source, analyze it, and review the data quality results. +This guide provides a quick tutorial on how to get started with DQOps using the web interface. You will add a data source, run data quality assessment, automatically configure data quality checks, +review the results and create an incident notification for data quality issues. ## Sample data -In the example, we will import a CSV file with a sample [public dataset Austin Crime Data](https://console.cloud.google.com/marketplace/details/city-of-austin/austin-crime). +In the example, we will import a CSV file with a sample fragment of the [public dataset of a fictitious ecommerce clothing site TheLook](https://console.cloud.google.com/marketplace/product/bigquery-public-data/thelook-ecommerce). -Next, we will run and review [basic statistics](../working-with-dqo/collecting-basic-data-statistics.md), -and automatically added profiling and monitoring [data quality checks](../dqo-concepts/definition-of-data-quality-checks/index.md). +Next, we will run data quality assessment, automatically configure data quality checks to find the most common data quality issues, +review the results and create an incident notification for data quality issues. Finally, we will review the data quality results on the [data quality dashboards](../dqo-concepts/types-of-data-quality-dashboards.md). @@ -23,9 +24,9 @@ Finally, we will review the data quality results on the [data quality dashboards 1. [Local installation](installation.md) -2. [Connect to a data source](add-data-source-connection.md) +2. [Add a data source connection and start data quality assessment](add-data-source-connection.md) -3. [Review the initial results and run monitoring checks](review-results-and-run-monitoring-checks.md) +3. [Review results from data quality assessment, automatically configure data quality checks and create incident notification](review-results-and-run-monitoring-checks.md) 4. [Review the results on data quality dashboards](review-results-on-dashboards.md) diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 70adc563d2..06ecf26fb6 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -149,7 +149,7 @@ dqo> settings apikey set Open the DQOps User Interface Console in your browser by CTRL-clicking on the link displayed on the command line (for example http://localhost:8888) or by copying the link. You will see the welcome screen as shown below. -![DQOps user interface](https://dqops.com/docs/images/getting-started/dqops-user-interface.png){ loading=lazy; width="1200px" } +![DQOps user interface](https://dqops.com/docs/images/getting-started/dqops-user-interface2.png){ loading=lazy; width="1200px" } ## Next step @@ -160,7 +160,7 @@ Now that you have installed and opened DQOps, it is time to ## Questions and answers If you have any questions, here are a few of the most common questions we answered. -### What happens after the trial period +### What happens after the trial period? You can always use almost all the features of our open-source platform as a free user. However, we sponsor your [data quality data warehouse](../dqo-concepts/architecture/dqops-architecture.md#data-quality-data-warehouse), which powers the [data quality dashboards](../dqo-concepts/types-of-data-quality-dashboards.md). @@ -171,20 +171,20 @@ Your complimentary [data quality data warehouse](../dqo-concepts/architecture/dq will store the data quality results only for the first five tables that you monitor. Every other DQOps feature will continue to work without any implications. -### Do I need a DQOps Cloud account +### Do I need a DQOps Cloud account? You don't need a DQOps Cloud account to use DQOps. Without a DQOps Cloud account, you won't have access to data quality dashboards, making it difficult to demonstrate data reliability to business sponsors and data customers. -### How to install DQOps without Python +### How to install DQOps without Python? Starting [DQOps locally as a Python package](../dqops-installation/install-dqops-using-pip.md) is not the only option. Review other [installation options](../dqops-installation/index.md) to learn more. -### How to run DQOps on a server +### How to run DQOps on a server? Using our docker image is the preferred way to start a long-running DQOps instance that constantly monitors your data sources. Follow the installation guide for [running DQOps in Docker](../dqops-installation/run-dqops-as-docker-container.md) to learn more. -### How to activate user & role management +### How to activate user & role management? FREE and PERSONAL editions are limited to a single user who runs DQOps locally. You must [contact our sales team](https://dqops.com/contact-us/) and request a trial period for a TEAM or ENTERPRISE edition that provides [multi-user management](../working-with-dqo/access-management.md) and integration with your single sign-on authentication. diff --git a/docs/getting-started/review-results-and-run-monitoring-checks.md b/docs/getting-started/review-results-and-run-monitoring-checks.md index a212b23d68..f3a283e3fe 100644 --- a/docs/getting-started/review-results-and-run-monitoring-checks.md +++ b/docs/getting-started/review-results-and-run-monitoring-checks.md @@ -1,37 +1,107 @@ --- -title: Review initial data quality results and run monitoring checks +title: Review results from data quality assessment and automatically configure data quality checks --- -# Review initial data quality results and run monitoring checks -This guide will show you how to run the first data quality check, and how to review the first results and data quality issues in DQOps. +# Review results from data quality assessment and automatically configure data quality checks +This guide will show you how to review results from data quality assessment and automatically configure data quality checks to detect the most common data quality issues. +We will run those check, review the first results and create an incident notification for data quality issues. ## Overview -After [adding your first connection, and initiating automatic monitoring](add-data-source-connection.md), -we describe how to review the initial results from the basic statistics and profiling checks, as well as how to run monitoring checks. - -Once new tables are imported, DQOps automatically activates default profiling and monitoring checks which are scheduled to run daily at 12:00 a.m. +After [adding your first connection, and starting data assessment](add-data-source-connection.md), +we describe how to review the initial results from the basic statistics and profiling checks. Then, by using the DQOps [rule mining engine](../dqo-concepts/data-quality-rule-mining.md), we will automatically +configure (mine) data quality checks to find the most common data quality issues. Next, we will run monitoring checks, review results and create an incident notification for data quality issues. ## Review basic statistics results Basic statistics provides you with summary information about your tables and columns. This information is valuable in deciding which data quality checks and threshold levels should be set to monitor data quality. -In the previous step we have collected basic statistics for imported tables and columns using the Advisor. +The information from basic statistics is also used by [rule miner](../dqo-concepts/data-quality-rule-mining.md) +to propose a reasonable configuration of data quality checks. + +In the previous step we have collected basic statistics for imported tables and columns. To review the results: -1. Go to the **Profiling** section. Select the **austin_crime.csv** table on the tree view on the left. The results are displayed under **Basic data statistics** tab. +1. Go to the **Profiling** section. +2. Select the **orders.csv** table on the tree view on the left. +3. The results are displayed under the **Basic data statistics** tab. You can filter the columns by simply clicking on the sorting icon next to any column header. For detailed description of each column go to the [Basics statistics section](../working-with-dqo/collecting-basic-data-statistics.md). - ![Basic statistics results for austin crimes](https://dqops.com/docs/images/getting-started/austin-crimes-statistics4.png){ loading=lazy; width="1200px" } + ![Basic statistics results for orders](https://dqops.com/docs/images/getting-started/orders-statistics1.png){ loading=lazy; width="1200px" } -2. To view detailed statistics, click on the name of the column or navigate to the single column on the tree view. +To view detailed statistics for column, click on the name of the column or navigate to the single column on the tree view. - ![Basic statistics results for austin crimes - details](https://dqops.com/docs/images/getting-started/austin-crimes-address-column-statistics5.png){ loading=lazy; width="1200px" } +![Basic statistics results for orders - details](https://dqops.com/docs/images/getting-started/orders-column-statistics1.png){ loading=lazy; width="1200px" } + +## Propose a configuration of data quality checks using rule mining + +Setting up data quality rules and thresholds manually can be time-consuming, especially since most checks will show that everything is okay. +DQOps automates this process with its data quality [rule mining engine](../dqo-concepts/data-quality-rule-mining.md). The engine +analyzes data collected from basic statistics, sample values and results from profiling checks to propose check configurations for detecting common data quality issues. +It also suggests checks that will pass for the current data and serve as a baseline for detecting changes in data distribution over time. + +You can find more information about [the data quality rule mining in the concept section](../dqo-concepts/data-quality-rule-mining.md). + +### Navigating to the rule miner + +To navigate to the data quality rule miner. + +1. Select the **Profiling** section +2. Select the table of interest on the tree view on the left. +3. Click on the **Data quality rule mining** tab + +![Navigating to rule miner](https://dqops.com/docs/images/getting-started/navigating-to-rule-miner2.png){ loading=lazy; width="1200px" } + +The rule mining screen allows you to view and select the automatically proposed configuration of data quality checks. +DQOps proposes a list of data quality checks instantly upon entering the rule mining screen. +If any parameter configuration is changed, clicking the **Propose** button will generate a new proposal. +Use the **Apply** button to save (apply) the proposed check configuration. + +The rule miner shows proposed configurations as a list of checks divided by table, columns and check types. +The column names can be expanded to view the proposed checks. You can change the proposed data quality check +parameters and data quality rule thresholds before applying the proposed configuration. + +You can also customize the rule miner proposals with the following options: + +- **Filtering by check category, check name and column name:** You can use these filtering options no narrow down the +list of proposals by specifying check category, check name, or column name. +- **Error rate (% of rows):** The most critical configuration parameter of the rule miner to control its sensitivity. +It represents the percentage of errors that can contain invalid values. By default, it is set to 2%, but you can change this value. +DQOps will attempt to configure data quality checks to detect data quality issues in 2% of records containing the most outstanding values. +- **Default severity level:** This is used to set the initial severity level for proposed checks, which can be later modified individually. +- **Advanced parameters:** These parameters contains checkboxes that allows control how the rule mining engine handles + already configured data quality checks and select data quality check categories that you want to include + in the proposal. More information about [Rule mining parameters](../dqo-concepts/data-quality-rule-mining.md#rule-mining-parameters). + +### Automated data quality check configuration using rule miner +As an example, let's filter the initial proposition with the column name **gender**. + +Click on the **Propose** button to get the filtered proposition. + +![Rule miner gender column](https://dqops.com/docs/images/getting-started/rule-mining-proposition-gender-column1.png){ loading=lazy; width="1200px" } + +From the list of propositions, we will make the following modifications: + +- In the **Nulls** category, keep only the **profile_null_percent** check with suggested max_percent parameter to value of 0. +- In the **Uniqueness** category, adjust the **profile_distinct_count** check to have a range value of 2 to ensure 2 distinct values (M and F) are present. +- In the **Accepted Values** category, keep the **profile_text_found_in_set_percent** check with the suggested values M and F. +- In the **Text** and **PII** categories, deactivate all checks using the green toggle button. + +Click on the **Apply** button to apply the proposed check configuration. + +![Rule miner gender column proposition modification](https://dqops.com/docs/images/getting-started/rule-mining-proposition-modifications1.png){ loading=lazy; width="1200px" } +A popup window will appear, notifying you that the checks have been activated, and that you can run activated check by clicking on the **Confirm** button. + +![Rule miner proposition - run activated checks](https://dqops.com/docs/images/getting-started/rule-mining-proposition-run-activated-checks1.png){ loading=lazy; width="600px" } + +A notification will appear indicating that the job has begun, and the **Table quality status** tab will start flashing. + +If desired, you can use the rule miner to activate more checks for table or other columns. ## Review profiling checks results @@ -39,161 +109,249 @@ To review the results: [**Basic data statistics**](../working-with-dqo/collecting-basic-data-statistics.md). Profiling checks are also useful for exploring and experimenting with various types of checks and determining the most suitable ones for regular data quality monitoring. -In the previous step we have run profiling checks for imported tables and columns using the Advisor. Now, let's review the results. +In the previous step we have run several profiling checks suggested by the [rule mining](../dqo-concepts/data-quality-rule-mining.md). Now, let's review the results. ### Review Profiling Table quality status -**Table quality status** allows you to quickly evaluate the data quality status of your table. To navigate to the Table quality status, -click on **Profiling** section, the "austin_crime.csv" table on the tree view, and **Table quality status** tab. +**Table quality status** allows you to quickly evaluate the data quality status of your table. + +To navigate to the Table quality status: + +1. Click on the **Profiling** section +2. Select the **orders.csv** table on the tree view on the left. +3. Click on the **Table quality status** tab. -![Profiling Table quality status](https://dqops.com/docs/images/getting-started/profiling-table-quality-status2.png){ loading=lazy; width="1200px" } +![Profiling Table quality status](https://dqops.com/docs/images/getting-started/profiling-table-quality-status5.png){ loading=lazy; width="1200px" } -This tab provides you with an overview of the current data quality status of your table, including the number of -executed checks, and detailed results per table and columns grouped by check categories. +This tab provides an overview of the current data quality status of your table. It includes the number of +executed checks and detailed results per table and columns grouped by check categories. -At the bottom of the screen, you will find a table that displays the check results per category, and per -table and columns. +At the bottom of the screen, you will find a table that displays the check results per category, table, and column. -The colored boxes indicates the current or the highest severity status: green for a valid result, yellow for a warning, -orange for an error, and red for a fatal error. By clicking on the arrow next to the colored box, you can view a list -of checks that contribute to the result. +The colored boxes indicate the current or the highest severity status: green for a correct result, yellow for a warning, +orange for an error, red for a fatal error, and grey stripes for an execution error. -You can group the results by data quality dimension using the radio button and filter the results by time frame +You can click on the colored box to view a list of checks that contribute to the result. Hovering over the check name will provide more details. + +In our example, the "gender" column currently has an error severity status, which resulted from the failed checks in the **Accepted values**, **Nulls**, and **Uniqueness** categories. + +![Profiling Table quality status - details](https://dqops.com/docs/images/getting-started/profiling-table-quality-status-details2.png){ loading=lazy; width="1200px" } + +You can group the results by data quality dimension using the radio button and filter the results by time frame and severity status. +### Review check results on the Profiling check editor -### Navigate to the Profiling checks editor +We can also review more detailed check results and error samples on the **Profiling check editor**. -To navigate to the Profiling checks editor, click on **Profiling** section, the **austin_crime.csv** table on the tree view, and **Profiling checks** tab. +To navigate to the Profiling checks editor, click on the **gender** column on the tree view on the left +and select the **Profiling checks editor** tab. -![Profiling checks list](https://dqops.com/docs/images/getting-started/profiling-checks-list3.png){ loading=lazy; width="1200px" } +![Profiling checks list](https://dqops.com/docs/images/getting-started/profiling-checks-list6.png){ loading=lazy; width="1200px" } -Here you can view the list of all table profiling checks. Notice that the toggle button next to the name of the -default checks activated upon importing new tables have a light green color. If the check has -been activated manually, the toggle button will have a darker green color. +On the **Profiling checks editor** you can view the list of all table profiling checks. Notice that the toggle button next to the name of the +default checks activated upon importing new tables has a light green color. If the check has +been activated manually or through the rule miner, the toggle button will have a darker green color. -The icons located before the name of each check allow you to: activate and deactivate it, configure settings, run a check, -review results, and get more information about it. +The icons located before the name of each check allow you to: -For activated checks, notice a square next to the name indicating the results of the check runs initiated by the Advisor: +- Activate and deactivate it using the toggle button. +- Enable or disable check. +- Configure check settings. +- Get information if the check has activated scheduling. +- [Run a check](../working-with-dqo/run-data-quality-checks.md), +- Review results +- View the check description. -- green for a valid result -- yellow for a warning -- orange for an error -- red for a fatal error -- black for execution error +For activated checks, notice a square next to the name indicating the results of the check: -### Review the profiling checks results +- Green for a correct result. +- Yellow for a warning. +- Orange for an error. +- Red for a fatal error. +- Black for execution error. -To review the profiling checks results, click the **Results** icon to view more details of the results. +You can find more information about the check editor in the [User interface overview section of the documentation](../dqo-concepts/dqops-user-interface-overview.md#check-editor). -![Checking results](https://dqops.com/docs/images/getting-started/checking-results3.png){ loading=lazy; width="1200px" } +To review the checks results, click the **Results** icon. Let's review the results for the **profile_text_found_in_set_percent** +check in the **Accepted values** category. -A table will appear with more details about the run check. The check displayed Valid results with the actual value 1 000. +![Checking results](https://dqops.com/docs/images/getting-started/checking-results5.png){ loading=lazy; width="1200px" } -You can also review the results of other table checks on the list, as well as review the list of the column-level checks. -Just select the column of interest form the tree view on the left. +A table will appear with more details about the run check. The check displayed an Error result with the actual value of 99.599%, indicating that not +all values in the table are in the expected_values set defined as a check parameter. -On the list of checks you can activate other checks, change their thresholds and run them. +### Review the profiling checks results error samples -Note that some activated checks, for example column-level profile_nulls_count and profile_null_percent, do not have thresholds levels set. -For those check a [sensor](../dqo-concepts/definition-of-data-quality-sensors.md) will be executed, and you can view its result on Results details, **Sensor readouts** tab. -Based on the results, you can set the threshold for these checks. +To assist with identifying the root cause of errors and cleaning up the data, DQOps offers error sampling. +You can view representative examples of data that do not meet the specified data quality criteria by clicking on the +**Error sampling** tab in the results section. -### Default data profiling checks -To perform [initial data profiling](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md#initial-data-quality-kpi-score), -DQOps activates the following [default data quality checks](../dqo-concepts/data-observability.md) +![Checking error samples](https://dqops.com/docs/images/getting-started/checking-error-samples2.png){ loading=lazy; width="1200px" } -| Target | Check name | Description | -|--------|------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| -| table | [profile row count](../checks/table/volume/row-count.md) | Captures the row count of the table and identifies empty tables. | -| table | [profile data freshness](../checks/table/timeliness/data-freshness.md) | Measures data freshness of the table. | -| table | [profile column count](../checks/table/schema/column-count.md) | Retrieves the metadata of the monitored table from the data source and captures the column count. | -| column | [profile nulls count](../checks/column/nulls/nulls-count.md) | Counts null values in every column and detects incomplete columns that contain null values. | -| column | [profile nulls percent](../checks/column/nulls/nulls-percent.md) | Measures the percentage of null values in every column. | -| column | [profile not nulls count](../checks/column/nulls/not-nulls-count.md) | Counts not null values in every column and detects empty columns that contain only null values. | - +We can see that there are four values that do not match the set defined as a check parameter. -## Run monitoring checks +For additional information about error sampling, please refer to [the Data Quality Error Sampling documentation](../dqo-concepts/data-quality-error-sampling.md). +You can check there how to configure ID column that will help in identifying error sample or download error samples as a CSV file. +## Copy the verified profiling check to monitoring checks +To start monitoring your data using checks configured in the Profiling sections we can copy the configuration to monitoring checks. [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are standard checks that monitor the data quality of a -table or column. These checks create a single data quality result for the entire table or column. There are two categories -of monitoring checks: daily checks and monthly checks. When run multiple times per day, the **daily checks** store only -the most recent result for each day. **Monthly checks** store the most recent results for each month the data quality -checks were run. - -Now let's run monitoring checks. - -1. Go to the **Monitoring Checks** section, and select the **austin_crime.csv** table from the tree view and **Daily** tab. - - ![Monitoring checks section](https://dqops.com/docs/images/getting-started/monitoring-checks-section3.png){ loading=lazy; width="1200px" } - -2. Click the **Run check** icon next to daily_row_count check. This check verifies that the number of rows in the table - does not exceed the minimum accepted count set as the threshold level which is 1 in this case. - - You can read more about [issue severity levels in DQOps concepts section](../dqo-concepts/definition-of-data-quality-checks/index.md#issue-severity-levels). - - ![Running check](https://dqops.com/docs/images/getting-started/run-daily-row-count-check3.png){ loading=lazy; width="1200px" } - - A green square should appear next to the name of the checks indicating that the result of the run check is valid. - You can view the details by placing the mouse cursor on the green square or view more detail results by clicking the - **Results** icon - - You can run checks for your entire connection, schema, table, or column from the tree view on the left-hand side of - the screen. To do so, click on the three-dot icon and select the Run check option. Check more information how to - [Run checks from a tree view](../working-with-dqo/run-data-quality-checks.md#Run-checks-from-a-tree-view). - - You can activate and run other monitoring checks and adjust their thresholds. - -### Default daily monitoring checks -DQOps activates the following [daily monitoring checks](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) -on every table and column to apply [data observability](../dqo-concepts/data-observability.md) of the data source. - -| Target | Check name | Description | -|--------|-----------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| table | [daily row count](../checks/table/volume/row-count.md) | Captures the row count of the table every day and identifies empty tables. | -| table | [daily row count anomaly](../checks/table/volume/row-count-anomaly.md) | Detects day-to-day anomalies in the table volume. Raises a data quality issue when the increase or decrease in the row count is in the top 1% of most significant changes. | -| table | [daily row count change](../checks/table/volume/row-count-change.md) | Detects significant day-to-day changes in the table volume. Raises a data quality issue when the increase or decrease in the row count is greater than 10%. | -| table | [daily data freshness](../checks/table/timeliness/data-freshness.md) | Measures data freshness of the table. Raises a data quality issue when the data is outdated by 2 days. | -| table | [daily data staleness](../checks/table/timeliness/data-staleness.md) | Measures data staleness (the time since the last data loading) of the table. Raises a data quality issue when the table was not updated for 2 or more days. | -| table | [daily table availability](../checks/table/availability/table-availability.md) | Verifies that a table exists, can be accessed, and queried without errors. Detects corrupted tables and expired credentials to data sources. | -| table | [daily column count](../checks/table/schema/column-count.md) | Retrieves the metadata of the monitored table from the data source and counts the number of columns. | -| table | [daily column count changed](../checks/table/schema/column-count-changed.md) | Detects whether the number of columns in a table has changed since the last time the check (checkpoint) was run. | -| table | [daily column list changed](../checks/table/schema/column-list-changed.md) | Detects if the list of columns has changed since the last time the check was run. | -| table | [daily column list or order changed](../checks/table/schema/column-list-or-order-changed.md) | Detects whether the list of columns and the order of columns have changed since the last time the check was run. | -| table | [daily column types changed](../checks/table/schema/column-types-changed.md) | Detects if the column names or column types have changed since the last time the check was run. | -| column | [daily nulls count](../checks/column/nulls/nulls-count.md) | Counts null values in every column without raising any data quality issues. | -| column | [daily nulls percent](../checks/column/nulls/nulls-percent.md) | Measures the percentage of null values in every column without raising any data quality issues. | -| column | [daily nulls percent anomaly](../checks/column/nulls/nulls-percent-anomaly.md) | Measures the percentage of null values in every column and detects anomalous changes in the percentage of null value. Raises a data quality issue for the top 1% biggest day-to-day changes. | -| column | [daily not nulls percent](../checks/column/nulls/not-nulls-percent.md) | Measures the percentage of null values in every column without raising any data quality issues. | -| column | [daily nulls percent change](../checks/column/nulls/nulls-percent-change.md) | Detects significant day-to-day changes in the percentage of null values in every column. Raises a data quality issue when the increase or decrease in the percentage of null values is greater than 10%. | -| column | [daily distinct count anomaly](../checks/column/uniqueness/distinct-count-anomaly.md) | Counts distinct values in every column and detects anomalous changes in the percentage of null value. Raises a data quality issue for the top 1% biggest day-to-day changes of the count of distinct values. | -| column | [daily detected datatype in text changed](../checks/column/datatype/detected-datatype-in-text-changed.md) | Scans all values in a text column and detects the data type of all values in a column. Raises a data quality issue when the type of texts changes. For example, when a column contained always numeric values, but a text value was found. | -| column | [daily sum anomaly](../checks/column/anomaly/sum-anomaly.md) | Sums values in all numeric columns. Detects day-to-day anomalies in the sum of numeric values. Raises a data quality issue for the top 1% biggest day-to-day changes. | -| column | [daily mean anomaly](../checks/column/anomaly/mean-anomaly.md) | Calculates a mean (average) value in all numeric columns. Detects day-to-day anomalies of the mean of numeric values. Raises a data quality issue for the top 1% biggest day-to-day changes. | -| column | [daily column exists](../checks/column/schema/column-exists.md) | Reads the metadata of the monitored table and verifies that the column still exists in the data source. | -| column | [daily column type changed](../checks/column/schema/column-type-changed.md) | Detects if the data type of the column has changed since the last time it was retrieved. | - -For more [general information about checks, see the DQOps concepts](../dqo-concepts/definition-of-data-quality-checks/index.md) section. +table or column. These checks create a single data quality result for the entire table or column. When run multiple times per day, the **daily checks** store only +the most recent result for each day. + +To copy the configuration of checks activated in the Profiling section to Monitoring: + +1. Navigate to the **Monitoring checks** section. +2. Select the **orders.csv** table from the tree view on the left. +3. Click on the **Copy verified profiling checks** tab. + + Here you can see a proposition of checks based on checks that have been activated in the Profiling section. + To increase the number of propositions, you can either activate more profiling checks manually or use the data quality rule miner in the Profiling section. + + ![Copy verified profiling checks to monitoring](https://dqops.com/docs/images/getting-started/copy-verified-profiling-checks-to-monitoring1.png){ loading=lazy; width="1200px" } + +4. Click on the **Apply** button to activate the proposed monitoring checks. +5. A popup window will appear, notifying you that the checks have been activated and that you can run the activated check by clicking on the **Confirm** button. +6. A notification will appear indicating that the job has begun, and the **Table quality status** tab will start flashing. + +## Review Monitoring Table quality status + +**Table quality status** allows you to quickly evaluate the data quality status of your table. + +To navigate to the **Table quality** status of the **Monitoring checks** section: + +1. Click on **Monitoring** section +2. Select the **orders.csv** table on the tree view on the left. +3. Click on the **Table quality status** tab + +![Monitoring Table quality status](https://dqops.com/docs/images/getting-started/monitoring-table-quality-status1.png){ loading=lazy; width="1200px" } + +We can see that the **gender** column currently has an error severity status, which resulted from the failed checks in the **Accepted values**, **Nulls**, and **Uniqueness** categories. + +## Run monitoring checks + +Now let's run some more monitoring checks, which have been automatically activated upon importing the tables. + +In the **Monitoring checks** section, select the gender column to open the **Data quality check editor**. + +![Monitoring check editor](https://dqops.com/docs/images/getting-started/monitoring-check-editor.png){ loading=lazy; width="1200px" } + +Notice that the toggle button next to the name of the default checks activated upon importing new tables has a light green color. If the check has +been activated manually or through the rule miner, the toggle button will have a darker green color. + +To run a single check, click the **Run check** icon next to its name. To run all activated checks in a category, click the icon on the category row. +To run all activated checks in a table or column, click the icon in the top right corner of the Check Editor table. + +![Run monitoring checks](https://dqops.com/docs/images/getting-started/run-monitoring-checks1.png){ loading=lazy; width="1200px" } + +You can also run checks for your entire connection, schema, table, or column from the tree view on the left-hand side of +the screen. To do so, click on the three-dot icon and select the Run check option. Check more information on how to +[Run checks from a tree view](../working-with-dqo/run-data-quality-checks.md#run-checks-from-a-tree-view). + +Let's execute all the monitoring checks activated in the **demo_connection** by clicking on the three-dot icon and selecting the **Run check** option. +Click on **Run checks** in the popup. + +![Run monitoring checks from three view](https://dqops.com/docs/images/getting-started/run-monitoring-checks-from-three-view1.png){ loading=lazy; width="600px"} + +Now you can check the Table status again or check the details results as before. ## View Tables summary -To review the overall health summary of your data, navigate to the Home screen and use the Tables and Columns summaries. +To review the overall health summary of your data, you can use the Tables and Columns summaries. These summaries are based on previously executed monitoring and partition data quality checks, excluding the results from profiling checks. -To access **Tables summary**, click on the DQOps logo in the top left corner and select the **Tables** tab. +To access the **Tables summary**, click on the DQOps logo in the top left corner and select the **Data quality summary** tab and **Tables** subtab. -![Navigating to Tables summary](https://dqops.com/docs/images/getting-started/austin-crime-table-summary.png){ loading=lazy; width="1200px" } +![Navigating to the Tables summary](https://dqops.com/docs/images/getting-started/table-summary2.png){ loading=lazy; width="1200px" } You can view a list of tables with measured total data quality KPI and a breakdown of KPIs calculated from executed checks -categorized by specific dimensions (e.g., completeness, validity, consistency). Hovering over the KPIs value will display +categorized by specific dimensions (e.g., completeness, validity, consistency). Hovering over the KPIs value will display a tooltip containing more details. -The calculated KPIs in our example are at 100% because we ran one monitoring check that gave a valid result. +The summary **Data quality KPI** calculated for the orders.csv table in our example is 94%. +You can also check Column summaries by clicking on the **Column** subtab. + +For a deeper dive into [Tables and Columns summaries, see the DQOps concepts section](../dqo-concepts/dqops-user-interface-overview.md#home-screen). + +## Data quality incidents +The failure of the data quality check is a data quality issue. To prevent overwhelming the support team with numerous individual data quality issues, +DQOps groups similar data quality issues that share the same properties into data quality incident. +For more information on incidents, please refer to the [Data quality incident management, grouping and notifications section](../dqo-concepts/grouping-data-quality-issues-to-incidents.md). + +### View Incidents summary +The **Incidents summary** provides an overview of all incidents created in DQOps, which are groupings of similar data quality issues. +To access the **Incidents summary**, click on the DQOps logo in the top left corner and select the **Incidents summary** tab. + +![Navigating to the Incidents summary](https://dqops.com/docs/images/getting-started/incidents-summary2.png){ loading=lazy; width="1200px" } + +The **Incidents summary** screen gives an overview of data quality incidents categorized by either check category or data quality dimension. +You can easily switch between these groupings using the provided radio buttons. + +On the summary screen for Incidents, there are two sections displaying the summaries of **Open** and **Acknowledged** +incidents. Each section is further divided into three blocks based on severity level: Warnings, Errors, and Fatal errors. +Within each block, you will find a summary of new incidents from the last 2 months, along with detailed data showing the +number of incidents detected in the last 24 hours, last 7 days, current month, and previous month. + +Below the **Open** and **Acknowledged** incidents summaries, there is a table listing incidents grouped by the selected +check category or the quality dimension, depending on your grouping selection. + +The DQOps user interface overview section provides [a more detailed description of the **Incident summary** screen](../dqo-concepts/dqops-user-interface-overview.md#incidents_summary). + +### View an incident details +To view and manage incidents that occur during data quality monitoring, navigate to the **Incidents** section. +You can also access the Incidents section by clicking on the check category link or table name on the Incidents summary screen. + +Let's open the Incidents section displaying incidents in the Accepted values by clicking on the **Accepted values** lin. + +![Open the Accepted values incidents](https://dqops.com/docs/images/getting-started/open-the-accepted-values-incidents1.png){ loading=lazy; width="1200px" } + +On the left side of the **Incidents** section screen, there is a list displaying the connections and the number of incidents that have +occurred for each one. On the right panel, you can view incidents for the connections you have selected, and you can +filter, sort, and view detailed information about the incidents. You can find [more information about managing incidents in the Working with DQOps section](../working-with-dqo/managing-data-quality-incidents-with-dqops.md#view-and-manage-incidents). + +Click on the Data quality issue grouping column to view more details of the incident. + +![Open the incident details](https://dqops.com/docs/images/getting-started/open-the-incident-details1.png){ loading=lazy; width="1200px" } + +The detailed data quality incident screen shows a list of data quality issues that are grouped in the incident and allows you to +filter and sort them. + +![Incident details screen](https://dqops.com/docs/images/getting-started/incident-details-screen2.png){ loading=lazy; width="1200px" } + +The upper right corner of the Incident details screen provides access to several helpful actions: + +- **Disable checks for the incident**: This button allows you to temporarily disable the check responsible for the incident. Disabling a check can be helpful when you are actively working to resolve the underlying issue. +- **Reconfigure rule thresholds**: Clicking on this button will decrease the rule threshold for the data quality check that caused the incident by 30% to reduce the number of data quality issues. +- **Change incident configuration**: This button opens the **Incident configuration** screen for the table where the incident originated. This allows you to manage incident settings, such as grouping and severity levels. +- **Configure notification for this incident**: This button allows you to create a new or modify the existing notification filter for this incident. + +Follow the link to learn [more about the Incidents details screen](../working-with-dqo/managing-data-quality-incidents-with-dqops.md#view-details-of-the-incident) + +### Create a notification for the incident +To receive notifications for specific data quality incidents, you can create or edit notification filters. +To do this, click on the envelope icon located in the top right corner of the incident details screen. + +![Incident details screen](https://dqops.com/docs/images/getting-started/incident-details-envelope-icon1.png){ loading=lazy; width="1200px" } + +If no notification has ever been configured for the incident, you will see a popup message informing you to create a new notification configuration. + +![Creating notification for the incident](https://dqops.com/docs/images/getting-started/creating-notification-for-the-incident1.png){ loading=lazy } + +Once approved, you will be redirected to the page to create the new incident notification filter. +The configuration form will be partially filled based on the incident's data. The details of the filter configuration can be +found in the section [Notification filters](../dqo-concepts/grouping-data-quality-issues-to-incidents.md#notification-filters) below. + +![Configuring notification for the incident](https://dqops.com/docs/images/getting-started/configure-new-notification-for-the-incident1.png){ loading=lazy; width="1200px" } + +Add a name for the notification and optional message. Modify the target filters and fill in the **email address** or **webhook URL** for the appropriate status fields. You can also use multiple email addresses separated by the +comma character (,). Mixing email addresses with webhook addresses is also allowed if you use commas between them. + +After configuring the notification, click on the **Save** button to save the changes. -For a deeper dive into [Tables and Columns summaries, see the DQOps concepts section](../dqo-concepts/dqops-user-interface-overview.md#home-screen) +Learn more about [the configuration of the notification filtering in the Concept section](../dqo-concepts/grouping-data-quality-issues-to-incidents.md#incident-notifications). ## Next step -Now that you have reviewed the initial results from basic statistics and profiling checks and run monitoring checks, +Now that you have reviewed the initial results from basic statistics, configured checks using rule miner, run monitoring checks, and created incident notification, you can [review the results on the dashboards](review-results-on-dashboards.md). diff --git a/docs/getting-started/review-results-on-dashboards.md b/docs/getting-started/review-results-on-dashboards.md index 42737b6b47..9f3dc14e73 100644 --- a/docs/getting-started/review-results-on-dashboards.md +++ b/docs/getting-started/review-results-on-dashboards.md @@ -2,77 +2,104 @@ title: Review data quality results on data quality dashboards --- # Review data quality results on data quality dashboards -This guide will show how to use the data quality dashboards provided by DQOps, and how to review your first data quality KPI score after profiling the data. +This guide will show how to use the data quality dashboards provided by DQOps and how to review your first data quality KPI score after profiling the data. ## Overview -In the [previous step](review-results-and-run-monitoring-checks.md) we reviewed -the initial results from the basic statistics and profiling checks, as well as run one monitoring check. Now let's review -the results on the data quality dashboards. +In the [previous step](review-results-and-run-monitoring-checks.md), we reviewed the results from the data quality assessment and +automatically configured data quality checks to detect the most common data quality issues. +We ran those check, reviewed the first results, and created an incident notification for data quality issues. +Now, let's review the results on the data quality dashboards. -In DQOps, you can choose from a variety of built-in dashboard groups. -You can read more about the [data quality dashboards](../dqo-concepts/types-of-data-quality-dashboards.md) -in the DQOps concepts section. +In DQOps, you can choose from a variety of built-in dashboard groups. You can read more +about the [data quality dashboards](../dqo-concepts/types-of-data-quality-dashboards.md) in the DQOps concepts section. ## Prerequisite -To be able to display results on data quality dashboards you need to synchronize locally stored results with your DQOps Cloud account. -To synchronize all the data just click on the **Synchronize** button in the upper right corner of the navigation bar. +To display results on data quality dashboards, you need to synchronize locally stored results with your DQOps Cloud account. +To synchronize all the data, click the **Synchronize** button in the upper right corner of the navigation bar. -## Review the total number of issues raised by the profiling checks +!!! info "Complimentary data quality data warehouse for FREE accounts" -To see how many issues have been raised by profiling checks, you can use the **Data profiling issues count** dashboard located in the **Profiling** group. -To access it, toggle button **Show advanced dashboards** at the top of the tree view, go to the **Data Quality Dashboard** section and select the **Profiling issues count per check** dashboard from the tree view on the left. + DQOps provides a complimentary [data quality data lake](../dqo-concepts/architecture/dqops-architecture.md#data-quality-data-lake) + and [data quality data warehouse](../dqo-concepts/architecture/dqops-architecture.md#data-quality-data-warehouse) for users + using a **FREE** license of DQOps Cloud. + + Due to capacity constraints, a limited number of new users who + [register a FREE DQOps Cloud account](https://cloud.dqops.com/registration) are granted access to the + data warehouse. This allows them to synchronize data quality results with the data quality data warehouse and + use the data quality dashboards for free. New grants are released every day. + +## Navigating dashboards + +To view dashboards, simply go to the **Data quality dashboard** section, and select the dashboard of interest from the tree +view on the left. There are several groups and subgroups of data quality dashboards dedicated to analyzing results from +data quality checks. -Once you are on the **Profiling issues count per check** dashboard, you will be able to review a summary of -executed tests (checks) and their results, categorized by type of check. You can filter the data on the dashboard simply -by clicking on the filters on top part of the dashboard, as well as clicking on the name of the connection, schema, data group etc. +![Navigating to dashboards section](https://dqops.com/docs/images/getting-started/navigating-to-dashboards-section2.png){ loading=lazy; width="1200px" } -![Total issue details dashboard](https://dqops.com/docs/images/getting-started/profiling-issues-count-per-check-dashboard2.png){ loading=lazy; width="1200px" } +In DQOps, the dashboards are divided into two groups: basic and advanced dashboards. -In our example, we ran 56 profiling checks (2 table checks and 3 checks for each of the 18 columns) and none of them failed. -(A warning is not considered as a failed check.) In the **Results per check** table you can see the results broken down by check name. +The toggle button **Show advanced dashboards** at the top of the tree view allows you to toggle between +displaying only basic or both basic and more advanced dashboards. -## Review the total number of issues raised by the monitoring checks +## Review the Current column status -To review the issues raised by the monitoring checks, simply select the **Data quality issue count per check** dashboard -in the **Monitoring** group. +**Current status** dashboards allow data engineers and data owners to quickly evaluate the data quality of monitored +tables and columns. The dashboards display a color-coded status that indicates the severity level detected by run +checks. When the status is green, it means that the monitored column has no data quality issues. However, if the status +is yellow, orange, or red, it indicates that some issues were detected. The dashboard also displays the number +of detected issues per severity threshold, making it easier to identify and address tables and columns with issues. -![Total issue details dashboard](https://dqops.com/docs/images/getting-started/monitoring-issue-count-per-check-dashboard2.png){ loading=lazy; width="1200px" } +On top of the **Current column status** dashboard, you can review a summary of executed checks and +their results, categorized by severity level. +You can filter the data on the dashboard simply by clicking on the filters on the top part of the dashboard, +as well as clicking on the name of the connection, schema, data group, etc. -Here we can see the correct result from one daily_row_count check that we executed previously. +![Current column status](https://dqops.com/docs/images/getting-started/current-column-status-dashboard1.png){ loading=lazy; width="1200px" } -## Review the data quality KPIs +In our example, we ran 50 monitoring checks and reported 3 issues in error severity level. +The results are broken down by column name in the **Column** table at the bottom of the dashboard. + +## Review the Data quality KPIs Data quality KPIs are the calculated percentage of passed data quality checks. -Data quality KPIs dashboards show high-level data quality KPIs, aggregated on a macro scale that can be shared at a corporate level. -To read more about [data quality KPIs, check DQO concept section](../dqo-concepts/definition-of-data-quality-kpis.md). +Data quality KPIs dashboards show high-level data quality KPIs aggregated on a macro scale that can be shared at a corporate level. +To read more about [data quality KPIs, check DQOps concept section](../dqo-concepts/definition-of-data-quality-kpis.md). Data quality KPIs dashboards review key data metrics per connection, [data quality dimensions](../dqo-concepts/data-quality-dimensions.md), [check category](../dqo-concepts/definition-of-data-quality-checks/index.md#categories-of-checks) and [data grouping](../dqo-concepts/measuring-data-quality-with-data-grouping.md). -To review the summary KPIs of all run checks use the **KPIs scorecard - summary** dashboards. +To review the summary KPIs of all run checks use the **KPIs scorecard - summary** dashboard located in the Data quality KPIs group. -In the **Profiling** group, select the **Profile data quality KPIs** section and select the -**Profiling KPIs scorecard - summary** dashboard. +![KPIs scorecard dashboard](https://dqops.com/docs/images/getting-started/monitoring-kpis-scorecard-dashboard1.png){ loading=lazy; width="1200px" } -![KPIs scorecard dashboard](https://dqops.com/docs/images/getting-started/profiling-kpis-scorecard-dashboard3.png){ loading=lazy; width="1200px" } - -The calculated percentage of passed data quality checks (KPI) in our example is 100%. On other dashboards in KPIs category, -you can also review KPIs results broken down by table, data quality dimension or check category. +Our example's calculated percentage of passed data quality checks (KPI) is 90.9%. On other dashboards in KPIs category, +you can also review KPIs results broken down by table, months and days, data quality dimension, or check category. + +## Review the Current validity issues on columns +The **Data Quality Dimensions** group contains dashboards dedicated to different data quality dimensions, +such as availability, completeness, timeliness, consistency, and validity. + +**Current validity issues on columns** is a representative of the **Validity** subgroup. This dashboard summarizes +results from the most recently executed checks categorized in the validity dimension. + +![Current validity issues on columns dashboard](https://dqops.com/docs/images/getting-started/monitoring-current-validity-issues-on-columns-dashboard2.png){ loading=lazy; width="1200px" } + +Here, we can see that only one Validation check was executed, which resulted in an error. The **Column** table at the +bottom of the dashboard breaks down the results by column and check names. The check that resulted in an error is **daily_text_found_in_set_percent**, +and you can see the actual and expected values of the check. You have completed our quick tutorial, and now you know how to add connections, run basic statistics and data quality checks, as well as view results. ## Next steps +- Check the [Categories of data quality checks section ](../categories-of-data-quality-checks/index.md) to learn how the types of popular data quality + checks are divided into categories and what categories of checks are supported by DQOps - We have provided a variety of use cases that use openly available datasets from [Google Cloud](https://cloud.google.com/datasets) - to help you in using DQOps effectively. You can find the [full list of use cases here](../examples/index.md). -- DQOps allows you to keep track of the issues that arise during data quality monitoring and - send alert notifications directly to Slack. - Learn more about [incidents](../working-with-dqo/managing-data-quality-incidents-with-dqops.md) and - [Slack notifications](../integrations/slack/configuring-slack-notifications.md). + to help you use DQOps effectively. You can find the [full list of use cases here](../examples/index.md). - The data in the table often comes from different data sources and vendors or is loaded by different data pipelines. - Learn how [data grouping in DQOps](../working-with-dqo/set-up-data-grouping-for-data-quality-checks.md) - can help you calculate separate data quality KPI scores for different groups of rows. + Learn how [data grouping in DQOps](../working-with-dqo/set-up-data-grouping-for-data-quality-checks.md) + can help you calculate separate data quality KPI scores for different groups of rows. diff --git a/docs/images/border-website6.png b/docs/images/border-website6.png new file mode 100644 index 0000000000..8072238cad Binary files /dev/null and b/docs/images/border-website6.png differ diff --git a/docs/integrations/slack/configuring-slack-notifications.md b/docs/integrations/slack/configuring-slack-notifications.md index fb4dd7ca07..865568b623 100644 --- a/docs/integrations/slack/configuring-slack-notifications.md +++ b/docs/integrations/slack/configuring-slack-notifications.md @@ -59,22 +59,27 @@ Slack application supports configuration of multiple webhooks within a single ap ## Webhooks configuration in DQOps -There are two ways to configure webhooks in the DQOps platform. The first method is using the default webhooks +There are two ways to configure notification webhooks in the DQOps platform. The first method is using the default webhooks configuration available under the Configuration menu. -The second method is configuring webhooks on a connection level under the Notifications and Incidents tab, which is presented -in the Configuration steps in DQOps section. +The second method is configuring webhooks on a connection level under the **Notifications** tab, which is presented in the Configuration steps in DQOps section. -DQOps uses the default webhooks configuration by default. If the connection has a different webhooks configuration, this configuration is used. +DQOps uses the global notification configuration by default. If the connection notifications is set with the different webhook addresses, it has higher priority for utilisation. +The next steps only show the configuration of the default webhooks in the global incident notifications. +[The comprehensive guide](../../dqo-concepts/grouping-data-quality-issues-to-incidents.md) presents the configuration of the incidents and the notifications on a deeper level. -### Configure Default webhooks configuration +### Configure default webhooks configuration -You can configure default webhooks configuration for all connections. -Simply go to the **Configuration** section and select the **Default webhooks** from the tree view on the left. Add the -Webhook URLs generated in Slack to appropriate input fields. +The default webhooks configuration is applied for all connections. +Simply go to the **Configuration** section and select the **Global incident notifications** from the tree view on the left. +Edit the notification configuration named **default**. -![Default webhook configuration](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configuring-slack-notifications/default-webhooks-page.png) +![Connection notifications configuration](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/global-incident-notifications.png){ loading=lazy; width="1200px" } + +Add the Webhook URLs generated in Slack to appropriate input fields. + +![Global incident notifications default](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/global-incident-notification-default.png){ loading=lazy; width="1200px" } ### Configure Webhooks on connection level @@ -83,19 +88,19 @@ To configure webhooks on a connection level in DQOps, follow the steps below. 1. Open the DQOps application and go to the **Data Sources** section. - ![Navigating to the Incidents section](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configuring-slack-notifications/dqo-incidents.png) - -2. In the tree view on the left, select the data source of interest, and select the **Incidents and Notifications** tab. +2. In the tree view on the left, select the data source of interest, and go to the **Notifications** tab. The URL path part can be also used : /sources/connection/**[connection_name]**/incidents Remember to replace [connection_name] with your connection name, created in DQOps application. -3. In the **Webhooks for notifications of an incident state change** section, paste the Webhook URL that you get in the Slack app into the “A new incident was opened (detected)” field. + ![Global incident notifications default](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/connection-notification-configuration.png){ loading=lazy; width="1200px" } -4. Save the changes by clicking the **Save button**. Now each time a new incident is open you will receive a notification in Slack. +3. In the **Addresses for notifications of an incident state change** section, paste the Webhook URL that you get in the Slack app into the “**Open:** A new incident was opened (detected)” field. + + ![Configuring webhooks](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configuring-slack-notifications/webhook-fields2.png) - ![Configuring webhooks](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configuring-slack-notifications/webhook-fields.png) +4. Save the changes by clicking the **Save button**. Now each time a new incident is open you will receive a notification in Slack. !!! tip "Use another statuses" diff --git a/docs/integrations/webhooks/index.md b/docs/integrations/webhooks/index.md index 0949adb1f0..0d337f11d1 100644 --- a/docs/integrations/webhooks/index.md +++ b/docs/integrations/webhooks/index.md @@ -16,23 +16,26 @@ A notification contains the most important information about an incident you may ## Configure notifications -You can use Webhooks to receive notifications for new data quality incidents or changes in the incident status. +You can receive a notification with use of **webhooks** to receive notifications for new data quality incidents or changes in the incident status. -To configure Webhooks, follow these steps: +To configure a webhook notification, follow these steps: -1. Go to the **Data Sources** section. -2. Select the relevant data source. -3. Select the **Incidents And Notifications** tab. -4. Enter the Webhooks URL in the specified input fields and save your changes using the **Save** button. +1. Go to the **Configuration** section. +2. Open the **Global incident notifications** in tree and select the **default**. + + ![Connection notifications configuration](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/global-incident-notifications.png){ loading=lazy; width="1200px" } + +3. Enter the Webhooks URL in the specified input fields and save your changes using the **Save** button. + + ![Global incident notifications default](https://dqops.com/docs/images/concepts/grouping-data-quality-issues-to-incidents/global-incident-notification-default.png){ loading=lazy; width="1200px" } -![Configuring webhooks](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configuring-webhooks.png) ## Types of webhooks configuration Configuration of webhooks in DQOps platform can be done in two different ways. -First, is the default webhooks configuration available under Configuration menu. -Second, is to configure webhooks on a connection level under the Notifications and Incidents tab. +First, is to configure webhooks on a connection level under the Notifications and Incidents tab. +Second, is the default webhooks configuration available under Configuration menu. DQOps uses both of them when preparing notifications. When the connection does not set the webhook link on it's level, @@ -65,6 +68,7 @@ The fields of a JSON payload include the following: | **failed_checks_count** | The total number of failed data quality checks that were seen when the incident was raised for the first time. | String | | **issue_url** | The link (URL) to a ticket in an external system that is tracking this incident. | String | | **status** | Possible values for incident status include open, acknowledged, resolved and muted. | String | +| **message** | Additional message of the notification message configured by user from the filtered notifications. Default notificaitons does not contain such field. | String | | **text** | Notification text in Markdown format that contains the most important fields from the payload. | String | @@ -85,6 +89,7 @@ The json object presents as below: "highest_severity": 3, "failed_checks_count": 10, "status": "acknowledged", + "message": "An informative message about the incident", "text": "> *The incident in table has been acknowledged.* \n> \n> *First seen*: 2023-09-04 12:13:19 (GTM+2) \n> *Last seen*: 2023-09-04 14:35:51 (GTM+2) \n> *Quality dimension*: Reasonableness \n> *Check category*: volume \n> *Highest severity*: fatal \n> *Total data quality issues*: 10 \n> \n> \n" } ``` @@ -96,7 +101,7 @@ Notification message can have the following status: - **Open**: This is the initial status of a new incident, which is automatically assigned by DQOps. The remaining statuses of the incidents (Acknowledged, Resolved, and Muted) have to be set manually. An open incident indicates that a data quality team together with the data owner or someone else responsible for identifying issues should take action to address the data quality issue. You can change an incident's status back to Open at any time. - **Acknowledged**: This status indicates that the incident has been acknowledged, and someone has begun to investigate the data quality issue. The Acknowledged status has to be manually marked in the Incident section. Usually, the data quality issues present at the data source level can be fixed by the data producer. Issues caused by a bug in the data pipeline or an ETL process should be fixed by the Data Engineering Team. - **Resolved**: This status indicates that the root cause of the issue has been identified and fixed. The resolved status has to be set manually in the Incident section. -- **Muted**: This status hides the incident from your list. By default, any data quality issues associated with the incident will be muted for 60 days. If an incident is muted, DQO will not create a new one for 60 days. You can change the duration for muted incidents, by simply clicking the **Configure button** and changing the **Incidents and Notifications** settings. Muting the incident might be useful, when, for example, vendor’s data has been intentionally changed in comparison to the previous version. +- **Muted**: This status hides the incident from your list. By default, any data quality issues associated with the incident will be muted for 60 days. If an incident is muted, DQOps will not create a new one for 60 days. You can change the duration for muted incidents, by simply clicking the **Configure button** and changing the **Incidents and Notifications** settings. Muting the incident might be useful, when, for example, vendor’s data has been intentionally changed in comparison to the previous version. ## Text field content and format @@ -109,6 +114,7 @@ Payload's text field is built with the use of the following data: - **Data quality dimension**: Name of the dimension. [Read more](../../dqo-concepts/data-quality-dimensions.md) - **Highest severity level**: A severity level from the data quality rule. [Read more](../../dqo-concepts/definition-of-data-quality-checks/index.md) - **Total data quality issues**: A value from failedChecksCount JSON field. +- **Message**: Additional message of the notification message configured by user from the filtered notifications. Default notifications does not contain such field. - **Links**: Quick access links. **Links** @@ -154,6 +160,8 @@ Adding an Issue URL to an incident provides easy access to the issue in the tick ## Next steps +- Read about [detailed notifications configuration](../../dqo-concepts/grouping-data-quality-issues-to-incidents.md) - Now that you have learned about notifications, [set up Slack integration](../slack/configuring-slack-notifications.md) to receive them directly in Slack. - Learn how the [data quality incident workflow](../../dqo-concepts/grouping-data-quality-issues-to-incidents.md) is managed by DQOps by grouping similar data quality issues into data quality incidents. +- Learn about data quality incident management, grouping and notifications [grouping-data-quality-issues-to-incidents.md](../../dqo-concepts/grouping-data-quality-issues-to-incidents.md) \ No newline at end of file diff --git a/docs/reference/rules/Percentile.md b/docs/reference/rules/Percentile.md index c29936bb1b..793247cfd3 100644 --- a/docs/reference/rules/Percentile.md +++ b/docs/reference/rules/Percentile.md @@ -63,11 +63,19 @@ The rule definition YAML file *percentile/anomaly_differencing_percentile_moving data_type: double required: true default_value: 0.5 + parameters: + degrees_of_freedom: 4 ``` +**Additional rule parameters** + +| Parameters name | Value | +|-----------------|-------| +|*degrees_of_freedom*|4| + **Rule source code** @@ -95,7 +103,7 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch # from datetime import datetime - from typing import Sequence + from typing import Sequence, Dict import numpy as np import scipy import scipy.stats @@ -119,6 +127,10 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch min_periods_with_readouts: int + class AnomalyConfigurationParameters: + degrees_of_freedom: float + + # rule execution parameters, contains the sensor value (actual_value) and the rule parameters class RuleExecutionRunParameters: actual_value: float @@ -126,6 +138,7 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch time_period_local: datetime previous_readouts: Sequence[HistoricDataPoint] time_window: RuleTimeWindowSettingsSpec + configuration_parameters: AnomalyConfigurationParameters # default object that should be returned to the dqo.io engine, specifies if the rule was passed or failed, @@ -157,6 +170,7 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch differences = np.diff(filtered) differences_std = float(scipy.stats.tstd(differences)) differences_mean = float(np.mean(differences)) + degrees_of_freedom = float(rule_parameters.configuration_parameters.degrees_of_freedom) last_readout = float(filtered[-1]) actual_difference = rule_parameters.actual_value - last_readout @@ -165,8 +179,8 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch threshold_lower = float(differences_mean) threshold_upper = float(differences_mean) else: - # Assumption: the historical data follows normal distribution - readout_distribution = scipy.stats.norm(loc=differences_mean, scale=differences_std) + # Assumption: the historical data follows t-student distribution + readout_distribution = scipy.stats.t(df=degrees_of_freedom, loc=differences_mean, scale=differences_std) one_sided_tail = rule_parameters.parameters.anomaly_percent / 100.0 / 2 threshold_lower = float(readout_distribution.ppf(one_sided_tail)) @@ -241,11 +255,19 @@ The rule definition YAML file *percentile/anomaly_differencing_percentile_moving data_type: double required: true default_value: 0.5 + parameters: + degrees_of_freedom: 4 ``` +**Additional rule parameters** + +| Parameters name | Value | +|-----------------|-------| +|*degrees_of_freedom*|4| + **Rule source code** @@ -297,6 +319,10 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch min_periods_with_readouts: int + class AnomalyConfigurationParameters: + degrees_of_freedom: float + + # rule execution parameters, contains the sensor value (actual_value) and the rule parameters class RuleExecutionRunParameters: actual_value: float @@ -304,6 +330,7 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch time_period_local: datetime previous_readouts: Sequence[HistoricDataPoint] time_window: RuleTimeWindowSettingsSpec + configuration_parameters: AnomalyConfigurationParameters # default object that should be returned to the dqo.io engine, specifies if the rule was passed or failed, @@ -335,6 +362,7 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch differences = np.diff(filtered) differences_std = float(scipy.stats.tstd(differences)) differences_mean = float(np.mean(differences)) + degrees_of_freedom = float(rule_parameters.configuration_parameters.degrees_of_freedom) last_readout = float(filtered[-1]) actual_difference = rule_parameters.actual_value - last_readout @@ -343,8 +371,8 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch threshold_lower = float(differences_mean) threshold_upper = float(differences_mean) else: - # Assumption: the historical data follows normal distribution - readout_distribution = scipy.stats.norm(loc=differences_mean, scale=differences_std) + # Assumption: the historical data follows t-student distribution + readout_distribution = scipy.stats.t(df=degrees_of_freedom, loc=differences_mean, scale=differences_std) one_sided_tail = rule_parameters.parameters.anomaly_percent / 100.0 / 2 threshold_lower = float(readout_distribution.ppf(one_sided_tail)) @@ -417,11 +445,19 @@ The rule definition YAML file *percentile/anomaly_stationary_percentile_moving_a data_type: double required: true default_value: 0.5 + parameters: + degrees_of_freedom: 4 ``` +**Additional rule parameters** + +| Parameters name | Value | +|-----------------|-------| +|*degrees_of_freedom*|4| + **Rule source code** @@ -473,6 +509,10 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch min_periods_with_readouts: int + class AnomalyConfigurationParameters: + degrees_of_freedom: float + + # rule execution parameters, contains the sensor value (actual_value) and the rule parameters class RuleExecutionRunParameters: actual_value: float @@ -480,6 +520,7 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch time_period_local: datetime previous_readouts: Sequence[HistoricDataPoint] time_window: RuleTimeWindowSettingsSpec + configuration_parameters: AnomalyConfigurationParameters # default object that should be returned to the dqo.io engine, specifies if the rule was passed or failed, @@ -510,13 +551,14 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch filtered = np.array(extracted, dtype=float) filtered_std = scipy.stats.tstd(filtered) filtered_mean = np.mean(filtered) + degrees_of_freedom = float(rule_parameters.configuration_parameters.degrees_of_freedom) if filtered_std == 0: threshold_lower = float(filtered_mean) threshold_upper = float(filtered_mean) else: - # Assumption: the historical data follows normal distribution - readout_distribution = scipy.stats.norm(loc=filtered_mean, scale=filtered_std) + # Assumption: the historical data follows t-student distribution + readout_distribution = scipy.stats.t(df=degrees_of_freedom, loc=filtered_mean, scale=filtered_std) one_sided_tail = rule_parameters.parameters.anomaly_percent / 100.0 / 2 threshold_lower = float(readout_distribution.ppf(one_sided_tail)) @@ -589,11 +631,19 @@ The rule definition YAML file *percentile/anomaly_stationary_percentile_moving_a data_type: double required: true default_value: 0.5 + parameters: + degrees_of_freedom: 4 ``` +**Additional rule parameters** + +| Parameters name | Value | +|-----------------|-------| +|*degrees_of_freedom*|4| + **Rule source code** @@ -645,6 +695,10 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch min_periods_with_readouts: int + class AnomalyConfigurationParameters: + degrees_of_freedom: float + + # rule execution parameters, contains the sensor value (actual_value) and the rule parameters class RuleExecutionRunParameters: actual_value: float @@ -652,6 +706,7 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch time_period_local: datetime previous_readouts: Sequence[HistoricDataPoint] time_window: RuleTimeWindowSettingsSpec + configuration_parameters: AnomalyConfigurationParameters # default object that should be returned to the dqo.io engine, specifies if the rule was passed or failed, @@ -682,13 +737,14 @@ The file is found in the *[$DQO_HOME](../../dqo-concepts/architecture/dqops-arch filtered = np.array(extracted, dtype=float) filtered_std = scipy.stats.tstd(filtered) filtered_mean = np.mean(filtered) + degrees_of_freedom = float(rule_parameters.configuration_parameters.degrees_of_freedom) if filtered_std == 0: threshold_lower = float(filtered_mean) threshold_upper = float(filtered_mean) else: - # Assumption: the historical data follows normal distribution - readout_distribution = scipy.stats.norm(loc=filtered_mean, scale=filtered_std) + # Assumption: the historical data follows t-student distribution + readout_distribution = scipy.stats.t(df=degrees_of_freedom, loc=filtered_mean, scale=filtered_std) one_sided_tail = rule_parameters.parameters.anomaly_percent / 100.0 / 2 threshold_lower = float(readout_distribution.ppf(one_sided_tail)) diff --git a/docs/reference/sensors/column/datetime-column-sensors.md b/docs/reference/sensors/column/datetime-column-sensors.md index 558b5de373..f1603d71dc 100644 --- a/docs/reference/sensors/column/datetime-column-sensors.md +++ b/docs/reference/sensors/column/datetime-column-sensors.md @@ -885,7 +885,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM(CASE WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL AND - {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), should_make_text_constant = false ) }} + {{ lib.render_regex(lib.render_target_column('analyzed_table'), lib.render_date_format_regex(parameters.date_format), wrap_with_quotes = false) }} THEN 1 ELSE 0 END diff --git a/docs/reference/sensors/column/numeric-column-sensors.md b/docs/reference/sensors/column/numeric-column-sensors.md index 0a3bc1ae52..82493c8276 100644 --- a/docs/reference/sensors/column/numeric-column-sensors.md +++ b/docs/reference/sensors/column/numeric-column-sensors.md @@ -341,8 +341,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -359,8 +359,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -377,8 +377,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -395,8 +395,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -413,8 +413,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -437,8 +437,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -455,8 +455,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -480,8 +480,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -498,8 +498,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -516,8 +516,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -534,8 +534,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -552,8 +552,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -90.0 AND {{ lib.render_target_column('analyzed_table') }} <= 90.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -90.0 OR {{ lib.render_target_column('analyzed_table') }} > 90.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -602,8 +602,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -620,8 +620,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -638,8 +638,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -656,8 +656,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -674,8 +674,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -698,8 +698,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -716,8 +716,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} @@ -741,8 +741,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -759,8 +759,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 AND {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -777,8 +777,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -795,8 +795,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -813,8 +813,8 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} >= -180.0 AND {{ lib.render_target_column('analyzed_table') }} <= 180.0 THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} < -180.0 OR {{ lib.render_target_column('analyzed_table') }} > 180.0 THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections_reference('analyzed_table') }} diff --git a/docs/reference/sensors/column/patterns-column-sensors.md b/docs/reference/sensors/column/patterns-column-sensors.md index 87fca691fc..9fc6ec3a04 100644 --- a/docs/reference/sensors/column/patterns-column-sensors.md +++ b/docs/reference/sensors/column/patterns-column-sensors.md @@ -313,10 +313,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP_CONTAINS(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), r"^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$") - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -360,9 +360,9 @@ The templates used to generate the SQL query for each data source supported by D ELSE 100.0 * SUM( CASE WHEN {{lib.render_target_column('analyzed_table')}} - ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + !~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -382,10 +382,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$' ) }} - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -405,10 +405,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , + WHEN NOT REGEXP_LIKE({{lib.render_target_column('analyzed_table')}} , '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -434,10 +434,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{lib.render_target_column('analyzed_table')}} ~ + WHEN NOT {{lib.render_target_column('analyzed_table')}} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -457,10 +457,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -488,9 +488,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$' + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -510,10 +510,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -533,10 +533,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), + WHEN NOT REGEXP(CAST({{ lib.render_target_column('analyzed_table') }} AS STRING), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[a-zA-Z0-9-.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) / COUNT({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -556,14 +556,14 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN + WHEN NOT ( {{ lib.render_target_column('analyzed_table') }} LIKE '%[-a-zA-Z0-9!#$%&''*+\/=?^_`{|}~]@%.__%' AND len({{ lib.render_target_column('analyzed_table') }}) - len(replace({{ lib.render_target_column('analyzed_table') }}, '@', '')) = 1 -- single use of @ char AND right({{ lib.render_target_column('analyzed_table') }}, len({{ lib.render_target_column('analyzed_table') }}) - charindex('@', {{ lib.render_target_column('analyzed_table') }})) NOT LIKE '%[^-a-zA-Z0-9.]%' -- domain check AND len(left({{ lib.render_target_column('analyzed_table') }}, charindex('@', {{ lib.render_target_column('analyzed_table') }}))) < 64 -- local part length AND {{ lib.render_target_column('analyzed_table') }} not like '%@.%' - THEN 0 - ELSE 1 + ) THEN 1 + ELSE 0 END ) / COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) END AS actual_value @@ -583,10 +583,10 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(100.0 * SUM( CASE - WHEN REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), + WHEN NOT REGEXP_LIKE(CAST({{ lib.render_target_column('analyzed_table') }} AS VARCHAR), '^[a-zA-Z0-9.!#$%&''*+\/=?^_`{|}~-]{0,63}[a-zA-Z0-9!#$%&''*+\/=?^_`{|}~-]@[-a-zA-Z0-9.]+[.][a-zA-Z]{2,4}$') - THEN 0 - ELSE 1 + THEN 1 + ELSE 0 END ) AS DOUBLE) / COUNT({{ lib.render_target_column('analyzed_table') }}) END @@ -2631,9 +2631,10 @@ The templates used to generate the SQL query for each data source supported by D SELECT SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER, {{ lib.render_target_column('analyzed_table') }}) IS NULL + THEN 1 + ELSE 0 END ) AS actual_value {{- lib.render_data_grouping_projections('analyzed_table') }} @@ -2935,7 +2936,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(UNIQUEIDENTIFIER,{{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3018,7 +3020,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3041,7 +3044,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3064,7 +3068,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -3087,7 +3091,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3162,7 +3167,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3192,7 +3198,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3215,7 +3222,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3238,7 +3246,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3261,7 +3270,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.quote_identifier(column_name) }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3284,7 +3294,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3345,7 +3356,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND SAFE.PARSE_DATE({{lib.render_date_format(parameters.date_format)}}, {{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -3368,7 +3380,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_TIMESTAMP ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3391,7 +3404,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS FALSE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format_regex(parameters.date_format)}}) IS FALSE THEN 1 ELSE 0 END @@ -3414,7 +3427,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND STR_TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3489,7 +3503,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3519,7 +3534,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE({{ lib.render_target_column('analyzed_table') }}::VARCHAR, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3542,7 +3558,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3565,7 +3582,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TO_DATE ({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3588,7 +3606,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY_CONVERT(DATETIME, {{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}}) IS NULL THEN 1 ELSE 0 END @@ -3611,7 +3630,8 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL + WHEN {{ lib.render_target_column('analyzed_table') }} IS NOT NULL + AND TRY(DATE_PARSE({{ lib.render_target_column('analyzed_table') }}, {{lib.render_date_format(parameters.date_format)}})) IS NULL THEN 1 ELSE 0 END @@ -3731,7 +3751,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v\'\-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v\'\-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), '^(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-])|([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{1})([.]))([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})([ \t\n\r\f\v''-.]?([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([ \t\n\r\f\v''-.]?)(([a-zA-ZżźćńółęąśäöüåáéěíôúůýčďťĺňŕřšžçâêîôûàèìòùëïãõŻŹĆŃÓŁĘĄŚÄÖÜÅÁÉĚÍÔÚŮÝČĎŤĹŇŔŘŠŽÇÂÊÎÔÛÀÈÌÒÙËÏÃÕ]{2,})?([.])?))?$') }} THEN 1 ELSE 0 END @@ -3981,9 +4001,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_CONTAINS({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4004,9 +4024,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4027,9 +4047,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE - THEN 0 - ELSE 1 + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4050,9 +4070,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_regex(lib.render_target_column('analyzed_table'), parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4073,9 +4093,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4102,9 +4122,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE - THEN 0 - ELSE 1 + WHEN {{ lib.render_target_column('analyzed_table') }} !~ {{ lib.render_regex(parameters.regex) }} IS TRUE + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4125,9 +4145,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE CAST(SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) AS DOUBLE) END AS actual_value @@ -4155,9 +4175,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4178,9 +4198,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4201,9 +4221,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4224,9 +4244,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} - THEN 0 - ELSE 1 + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4247,9 +4267,9 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 0.0 ELSE SUM( CASE - WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) - THEN 0 - ELSE 1 + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 END ) END AS actual_value @@ -4331,7 +4351,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4354,7 +4374,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) IS TRUE + WHEN REGEXP_MATCHES({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) IS FALSE THEN 1 ELSE 0 END @@ -4400,7 +4420,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -4429,7 +4449,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} IS TRUE + WHEN {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} IS FALSE THEN 1 ELSE 0 END @@ -4452,7 +4472,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END @@ -4482,7 +4502,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} ~ {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4505,7 +4525,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} REGEXP {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4528,7 +4548,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} RLIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4551,7 +4571,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT_BIG({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE 100.0 * SUM( CASE - WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.make_text_constant(parameters.regex) }} + WHEN NOT {{ lib.render_target_column('analyzed_table') }} LIKE {{ lib.render_regex(parameters.regex) }} THEN 1 ELSE 0 END @@ -4574,7 +4594,7 @@ The templates used to generate the SQL query for each data source supported by D WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN 100.0 ELSE CAST(100.0 * SUM( CASE - WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.make_text_constant(parameters.regex) }}) + WHEN NOT REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ lib.render_regex(parameters.regex) }}) THEN 1 ELSE 0 END diff --git a/docs/reference/sensors/column/whitespace-column-sensors.md b/docs/reference/sensors/column/whitespace-column-sensors.md index c616561770..3d4919e2e3 100644 --- a/docs/reference/sensors/column/whitespace-column-sensors.md +++ b/docs/reference/sensors/column/whitespace-column-sensors.md @@ -1977,8 +1977,7 @@ The templates used to generate the SQL query for each data source supported by D SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2116,8 +2115,8 @@ The templates used to generate the SQL query for each data source supported by D SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 - AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 END @@ -2292,8 +2291,7 @@ The templates used to generate the SQL query for each data source supported by D ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND {{ lib.render_target_column('analyzed_table') }} <> '' - AND TRIM({{ lib.render_target_column('analyzed_table') }}) = '' + AND TRIM({{ lib.render_target_column('analyzed_table') }}) IS NULL THEN 1 ELSE 0 END @@ -2450,7 +2448,7 @@ The templates used to generate the SQL query for each data source supported by D ELSE 100.0 * SUM( CASE WHEN {{ lib.render_target_column('analyzed_table')}} IS NOT NULL - AND LEN({{ lib.render_target_column('analyzed_table')}}) <> 0 + AND DATALENGTH({{ lib.render_target_column('analyzed_table')}}) <> 0 AND TRIM({{ lib.render_target_column('analyzed_table')}}) = '' THEN 1 ELSE 0 diff --git a/docs/reference/sensors/index.md b/docs/reference/sensors/index.md index ae762b9069..7d2e2afd85 100644 --- a/docs/reference/sensors/index.md +++ b/docs/reference/sensors/index.md @@ -67,6 +67,16 @@ title: Sensors +### **uniqueness** + +| Sensor name | Description | +|-------------|-------------| +|[*duplicate_record_count*](./table/uniqueness-table-sensors.md#duplicate-record-count)|Table sensor that executes a duplicate record count query.| +|[*duplicate_record_percent*](./table/uniqueness-table-sensors.md#duplicate-record-percent)|Table sensor that executes a duplicate record percent query.| + + + + ### **volume** | Sensor name | Description | @@ -150,6 +160,9 @@ title: Sensors + + + diff --git a/docs/reference/sensors/table/index.md b/docs/reference/sensors/table/index.md index 71d603ca25..7bf452deb9 100644 --- a/docs/reference/sensors/table/index.md +++ b/docs/reference/sensors/table/index.md @@ -66,6 +66,16 @@ title: Sensors/table +### **uniqueness** + +| Sensor name | Description | +|-------------|-------------| +|[*duplicate_record_count*](./uniqueness-table-sensors.md#duplicate-record-count)|Table sensor that executes a duplicate record count query.| +|[*duplicate_record_percent*](./uniqueness-table-sensors.md#duplicate-record-percent)|Table sensor that executes a duplicate record percent query.| + + + + ### **volume** | Sensor name | Description | diff --git a/docs/reference/sensors/table/uniqueness-table-sensors.md b/docs/reference/sensors/table/uniqueness-table-sensors.md new file mode 100644 index 0000000000..b368fefac0 --- /dev/null +++ b/docs/reference/sensors/table/uniqueness-table-sensors.md @@ -0,0 +1,831 @@ +--- +title: DQOps data quality uniqueness sensors +--- +# DQOps data quality uniqueness sensors +All [data quality sensors](../../../dqo-concepts/definition-of-data-quality-sensors.md) in the **uniqueness** category supported by DQOps are listed below. Those sensors are measured on a table level. + +--- + + +## duplicate record count +Table sensor that executes a duplicate record count query. + +**Sensor summary** + +The duplicate record count sensor is documented below. + +| Target | Category | Full sensor name | Source code on GitHub | +|--------|----------|------------------|-----------------------| +| table | uniqueness | `table/uniqueness/duplicate_record_count` | [*sensors/table/uniqueness*](https://github.com/dqops/dqo/tree/develop/home/sensors/table/uniqueness/) | + + +**Sensor parameters** + +| Field name | Description | Allowed data type | Required | Allowed values | +|------------|-------------|-------------------|-----------------|----------------| +|`columns`|A list of columns used for uniqueness record duplicate verification.|*string_list*|:material-check-bold:|| + + + + + + +**Jinja2 SQL templates** + +The templates used to generate the SQL query for each data source supported by DQOps is shown below. + +=== "BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE + WHEN SUM(duplicated_count) IS NULL THEN 0 + ELSE SUM(CASE WHEN duplicated_count > 1 THEN 1 ELSE 0 END) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS duplicated_count + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +___ + + + +## duplicate record percent +Table sensor that executes a duplicate record percent query. + +**Sensor summary** + +The duplicate record percent sensor is documented below. + +| Target | Category | Full sensor name | Source code on GitHub | +|--------|----------|------------------|-----------------------| +| table | uniqueness | `table/uniqueness/duplicate_record_percent` | [*sensors/table/uniqueness*](https://github.com/dqops/dqo/tree/develop/home/sensors/table/uniqueness/) | + + +**Sensor parameters** + +| Field name | Description | Allowed data type | Required | Allowed values | +|------------|-------------|-------------------|-----------------|----------------| +|`columns`|A list of columns used for uniqueness record duplicate verification.|*string_list*|:material-check-bold:|| + + + + + + +**Jinja2 SQL templates** + +The templates used to generate the SQL query for each data source supported by DQOps is shown below. + +=== "BigQuery" + + ```sql+jinja + {% import '/dialects/bigquery.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Databricks" + + ```sql+jinja + {% import '/dialects/databricks.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "DuckDB" + + ```sql+jinja + {% import '/dialects/duckdb.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST( ', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "MySQL" + + ```sql+jinja + {% import '/dialects/mysql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns) ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Oracle" + + ```sql+jinja + {% import '/dialects/oracle.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR(4000))') ~ ') IS NOT NULL') }} + ) analyzed_table + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "PostgreSQL" + + ```sql+jinja + {% import '/dialects/postgresql.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Presto" + + ```sql+jinja + {% import '/dialects/presto.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Redshift" + + ```sql+jinja + {% import '/dialects/redshift.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_suffix='::VARCHAR') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Snowflake" + + ```sql+jinja + {% import '/dialects/snowflake.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Spark" + + ```sql+jinja + {% import '/dialects/spark.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS STRING)') ~ ') IS NOT NULL') }} + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "SQL Server" + + ```sql+jinja + {% import '/dialects/sqlserver.sql.jinja2' as lib with context -%} + + {%- macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro -%} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) * 1.0 / SUM(records_number)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections('analyzed_table', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table + {{- lib.render_where_clause(indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + GROUP BY {{- extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +=== "Trino" + + ```sql+jinja + {% import '/dialects/trino.sql.jinja2' as lib with context -%} + + {% macro extract_in_list(values_list, column_prefix = none, column_suffix = none, separate_by_comma = false) %} + {%- set column_names = table.columns if values_list is none or (values_list | length()) == 0 else values_list -%} + {%- for item in column_names -%} + {{ (column_prefix) if column_prefix is not none -}} {{- lib.quote_identifier(item) -}} {{- (column_suffix) if column_suffix is not none -}} {{- ", " if not loop.last }} {{- "', ', " if separate_by_comma and not loop.last }} + {%- endfor -%} + {% endmacro %} + + SELECT + CASE WHEN SUM(distinct_records) IS NULL THEN 0 + ELSE (1 - SUM(distinct_records) / CAST(SUM(records_number) AS DOUBLE)) * 100.0 END + AS actual_value + {{- lib.render_data_grouping_projections_reference('grouping_table') }} + {{- lib.render_time_dimension_projection_reference('grouping_table') }} + FROM ( + SELECT COUNT(*) AS records_number, + COUNT(*) OVER (PARTITION BY {{ extract_in_list(parameters.columns) -}} ) AS distinct_records + {{- lib.render_data_grouping_projections_reference('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table_nested', indentation=' ') }} + FROM ( + SELECT + {{ extract_in_list(parameters.columns) -}} + {{- lib.render_data_grouping_projections('analyzed_table_nested', indentation=' ') }} + {{- lib.render_time_dimension_projection('analyzed_table_nested', indentation=' ') }} + FROM {{ lib.render_target_table() }} AS analyzed_table_nested + {{- lib.render_where_clause(table_alias_prefix = 'analyzed_table_nested', indentation=' ', extra_filter = 'COALESCE(' ~ extract_in_list(parameters.columns, column_prefix='CAST(', column_suffix=' AS VARCHAR)') ~ ') IS NOT NULL') }} + ) + GROUP BY {{ extract_in_list(parameters.columns) -}} {{- (", " ~ lib.render_grouping_column_names()) if (lib.data_groupings is not none and (lib.data_groupings | length()) > 0) or lib.time_series is not none }} + ) grouping_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} + ``` +___ + + + + +## What's next +- Learn how the [data quality sensors](../../../dqo-concepts/definition-of-data-quality-sensors.md) are defined in DQOps and what is the definition of all Jinja2 macros used in the templates +- Understand how DQOps [runs data quality checks](../../../dqo-concepts/architecture/data-quality-check-execution-flow.md), rendering templates to SQL queries diff --git a/docs/reference/yaml/ColumnDefaultChecksPatternYaml.md b/docs/reference/yaml/ColumnDefaultChecksPatternYaml.md index d0a8ada74c..8d6cccfd1c 100644 --- a/docs/reference/yaml/ColumnDefaultChecksPatternYaml.md +++ b/docs/reference/yaml/ColumnDefaultChecksPatternYaml.md @@ -15,13 +15,13 @@ The structure of this object is described below |---------------|---------------------------------|-----------|-------------|---------------|---------------| |`api_version`|DQOps YAML schema version|*string*| |dqo/v1| | |`kind`|File type|*enum*|*source*
*table*
*sensor*
*provider_sensor*
*rule*
*check*
*settings*
*file_index*
*dashboards*
*default_schedules*
*default_checks*
*default_table_checks*
*default_column_checks*
*default_notifications*
|default_column_checks| | -|[`spec`](./ColumnDefaultChecksPatternYaml.md#columndefaultcheckspatternspec)|The specification (configuration) of default data quality checks (data observability checks) that are applied on columns matching a pattern|*[ColumnDefaultChecksPatternSpec](./ColumnDefaultChecksPatternYaml.md#columndefaultcheckspatternspec)*| | | | +|[`spec`](./ColumnDefaultChecksPatternYaml.md#columnqualitypolicyspec)|The specification (configuration) of default data quality checks (data observability checks) that are applied on columns matching a pattern|*[ColumnQualityPolicySpec](./ColumnDefaultChecksPatternYaml.md#columnqualitypolicyspec)*| | | | ___ -## ColumnDefaultChecksPatternSpec +## ColumnQualityPolicySpec The default configuration of column-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on columns. This configuration serves as a data quality policy that defines the data quality checks that are verified on matching columns. diff --git a/docs/reference/yaml/ConnectionYaml.md b/docs/reference/yaml/ConnectionYaml.md index 8b28142792..32a6de8e2b 100644 --- a/docs/reference/yaml/ConnectionYaml.md +++ b/docs/reference/yaml/ConnectionYaml.md @@ -48,6 +48,7 @@ The structure of this object is described below |[`incident_grouping`](./ConnectionYaml.md#connectionincidentgroupingspec)|Configuration of data quality incident grouping. Configures how failed data quality checks are grouped into data quality incidents.|*[ConnectionIncidentGroupingSpec](./ConnectionYaml.md#connectionincidentgroupingspec)*| | | | |[`comments`](./profiling/table-profiling-checks.md#commentslistspec)|Comments for change tracking. Please put comments in this collection because YAML comments may be removed when the YAML file is modified by the tool (serialization and deserialization will remove non tracked comments).|*[CommentsListSpec](./profiling/table-profiling-checks.md#commentslistspec)*| | | | |[`labels`](./ConnectionYaml.md#labelsetspec)|Custom labels that were assigned to the connection. Labels are used for searching for tables when filtered data quality checks are executed.|*[LabelSetSpec](./ConnectionYaml.md#labelsetspec)*| | | | +|`advanced_properties`|A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value dictionary.|*Dict[string, string]*| | | | @@ -582,7 +583,7 @@ The structure of this object is described below |`data_group_name`|Data group name. This field accepts search patterns in the format: 'group_name_\*', '\*group', 'prefix\*suffix'.|*string*| | | | |`quality_dimension`|Quality dimension.|*string*| | | | |`check_category`|The target check category, for example: *nulls*, *volume*, *anomaly*.|*string*| | | | -|`check_type`|The target type of checks to run. Supported values are *profiling*, *monitoring* and *partitioned*.|*string*| | | | +|`check_type`|The target type of checks to run. Supported values are *profiling*, *monitoring* and *partitioned*.|*enum*|*profiling*
*monitoring*
*partitioned*
| | | |`check_name`|The target check name to run only this named check. Uses the short check name which is the name of the deepest folder in the *checks* folder. This field supports search patterns such as: 'profiling_\*', '\*_count', 'profiling_\*_percent'.|*string*| | | | |`highest_severity`|Highest severity.|*integer*| | | | diff --git a/docs/reference/yaml/LocalSettingsYaml.md b/docs/reference/yaml/LocalSettingsYaml.md index a25aec0a04..8dd6f1495e 100644 --- a/docs/reference/yaml/LocalSettingsYaml.md +++ b/docs/reference/yaml/LocalSettingsYaml.md @@ -33,11 +33,12 @@ The structure of this object is described below |---------------|---------------------------------|-----------|-------------|---------------|---------------| |`editor_name`|Editor name spec (VSC, Eclipse, Intellij)|*string*| | | | |`editor_path`|Editor path on user's computer|*string*| | | | -|`api_key`|Api key|*string*| | | | -|`disable_cloud_sync`|Disable synchronization with DQOps cloud|*boolean*| | | | +|`api_key`|DQOps CLoud API key|*string*| | | | +|`disable_cloud_sync`|Disable synchronization with DQOps Cloud|*boolean*| | | | |`instance_signature_key`|DQOps instance signature key used to sign keys. This should be a Base64 encoded binary key at a 32 bytes length.|*string*| | | | |`time_zone`|Default IANA time zone name of the server. This time zone is used to convert the time of UTC timestamps values returned from databases to a uniform local date and time. The default value is the local time zone of the DQOps server instance.|*string*| | | | |[`smtp_server_configuration`](./LocalSettingsYaml.md#smtpserverconfigurationspec)|SMTP server configuration for incident notifications.|*[SmtpServerConfigurationSpec](./LocalSettingsYaml.md#smtpserverconfigurationspec)*| | | | +|[`data_domains`](./LocalSettingsYaml.md#localdatadomainspecmap)|The dictionary containing the configuration of local data domains.|*[LocalDataDomainSpecMap](./LocalSettingsYaml.md#localdatadomainspecmap)*| | | | @@ -59,5 +60,34 @@ The structure of this object is described below +___ + +## LocalDataDomainSpecMap +Dictionary of local data domains. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|`self`||*Dict[string, [LocalDataDomainSpec](./LocalSettingsYaml.md#localdatadomainspec)]*| | | | + + + +___ + +## LocalDataDomainSpec +Specification of the local data domain. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|`display_name`|Data domain display name, which is a user-friendly name to be shown in the UI|*string*| | | | +|`enable_scheduler`|Enables the job scheduler for this domain.|*boolean*| | | | + + + ___ diff --git a/docs/reference/yaml/SensorDefinitionYaml.md b/docs/reference/yaml/SensorDefinitionYaml.md index 8c2391d374..eb3a22ff33 100644 --- a/docs/reference/yaml/SensorDefinitionYaml.md +++ b/docs/reference/yaml/SensorDefinitionYaml.md @@ -65,7 +65,7 @@ The structure of this object is described below |`display_name`|Field display name that should be shown as a label for the control.|*string*| | | | |`help_text`|Help text (full description) that will be shown to the user as a hint when the cursor is moved over the control.|*string*| | | | |`data_type`|Parameter data type.|*enum*|*string*
*boolean*
*integer*
*long*
*double*
*date*
*datetime*
*column_name*
*enum*
*string_list*
*integer_list*
*object*
| | | -|`display_hint`|UI control display hint.|*enum*|*textarea*
| | | +|`display_hint`|UI control display hint.|*enum*|*textarea*
*column_names*
| | | |`required`|True when the value for the parameter must be provided.|*boolean*| | | | |`allowed_values`|List of allowed values for a field that is of an enum type.|*List[string]*| | | | |`default_value`|The default value for a parameter in a custom check or a custom rule.|*string*| | | | diff --git a/docs/reference/yaml/TableDefaultChecksPatternYaml.md b/docs/reference/yaml/TableDefaultChecksPatternYaml.md index e010949b53..4cd49ddb6d 100644 --- a/docs/reference/yaml/TableDefaultChecksPatternYaml.md +++ b/docs/reference/yaml/TableDefaultChecksPatternYaml.md @@ -15,13 +15,13 @@ The structure of this object is described below |---------------|---------------------------------|-----------|-------------|---------------|---------------| |`api_version`|DQOps YAML schema version|*string*| |dqo/v1| | |`kind`|File type|*enum*|*source*
*table*
*sensor*
*provider_sensor*
*rule*
*check*
*settings*
*file_index*
*dashboards*
*default_schedules*
*default_checks*
*default_table_checks*
*default_column_checks*
*default_notifications*
|default_table_checks| | -|[`spec`](./TableDefaultChecksPatternYaml.md#tabledefaultcheckspatternspec)|The specification (configuration) of default data quality checks (data observability checks) that are applied on tables matching a pattern|*[TableDefaultChecksPatternSpec](./TableDefaultChecksPatternYaml.md#tabledefaultcheckspatternspec)*| | | | +|[`spec`](./TableDefaultChecksPatternYaml.md#tablequalitypolicyspec)|The specification (configuration) of default data quality checks (data observability checks) that are applied on tables matching a pattern|*[TableQualityPolicySpec](./TableDefaultChecksPatternYaml.md#tablequalitypolicyspec)*| | | | ___ -## TableDefaultChecksPatternSpec +## TableQualityPolicySpec The default configuration of table-level data quality checks that are enabled as data observability checks to analyze basic measures and detect anomalies on tables. This configuration serves as a data quality policy that defines the data quality checks that are verified on matching tables. diff --git a/docs/reference/yaml/TableYaml.md b/docs/reference/yaml/TableYaml.md index 0cf31941a6..0e84e226fb 100644 --- a/docs/reference/yaml/TableYaml.md +++ b/docs/reference/yaml/TableYaml.md @@ -51,6 +51,8 @@ The structure of this object is described below |[`labels`](./ConnectionYaml.md#labelsetspec)|Custom labels that were assigned to the table. Labels are used for searching for tables when filtered data quality checks are executed.|*[LabelSetSpec](./ConnectionYaml.md#labelsetspec)*| | | | |[`comments`](./profiling/table-profiling-checks.md#commentslistspec)|Comments used for change tracking and documenting changes directly in the table data quality specification file.|*[CommentsListSpec](./profiling/table-profiling-checks.md#commentslistspec)*| | | | |[`file_format`](./TableYaml.md#fileformatspec)|File format with the specification used as a source data. It overrides the connection spec's file format when it is set|*[FileFormatSpec](./TableYaml.md#fileformatspec)*| | | | +|`advanced_properties`|A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value dictionary.|*Dict[string, string]*| | | | +|[`source_tables`](./TableYaml.md#tablelineagesourcespeclist)|A list of source tables. This information is used to define the data lineage report for the table.|*[TableLineageSourceSpecList](./TableYaml.md#tablelineagesourcespeclist)*| | | | @@ -337,6 +339,7 @@ The structure of this object is described below |[`statistics`](./TableYaml.md#columnstatisticscollectorsrootcategoriesspec)|Custom configuration of a column level statistics collector (a basic profiler). Enables customization of the statistics collector settings when the collector is analysing this column.|*[ColumnStatisticsCollectorsRootCategoriesSpec](./TableYaml.md#columnstatisticscollectorsrootcategoriesspec)*| | | | |[`labels`](./ConnectionYaml.md#labelsetspec)|Custom labels that were assigned to the column. Labels are used for searching for columns when filtered data quality checks are executed.|*[LabelSetSpec](./ConnectionYaml.md#labelsetspec)*| | | | |[`comments`](./profiling/table-profiling-checks.md#commentslistspec)|Comments for change tracking. Please put comments in this collection because YAML comments may be removed when the YAML file is modified by the tool (serialization and deserialization will remove non tracked comments).|*[CommentsListSpec](./profiling/table-profiling-checks.md#commentslistspec)*| | | | +|`advanced_properties`|A dictionary of advanced properties that can be used for e.g. to support mapping data to data catalogs, a key/value dictionary.|*Dict[string, string]*| | | | @@ -855,5 +858,91 @@ The structure of this object is described below +___ + +## TableLineageSourceSpecList +List of source tables of the current table to build the data lineage report. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|`self`||*List[[TableLineageSource](./TableYaml.md#tablelineagesource)]*| | | | + + + +___ + +## TableLineageSource +Key object that identifies a source table by using the connection name, schema name and table name to identify. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|`connection`|Connection name|*string*| | | | +|`schema`|Schema name|*string*| | | | +|`table`|Table name|*string*| | | | + + + +___ + +## TableLineageSourceSpec +Data lineage specification for a table to identify a source table of the current table where this object is stored. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|`source_connection`|The name of a source connection that is defined in DQOps and contains a source table from which the current table receives data.|*string*| | | | +|`source_schema`|The name of a source schema within the source connection that contains a source table from which the current table receives data.|*string*| | | | +|`source_table`|The name of a source table in the source schema from which the current table receives data.|*string*| | | | +|`data_lineage_source_tool`|The name of a source tool from which this data lineage information was copied. This field should be filled when the data lineage was imported from another data catalog or a data lineage tracking platform.|*string*| | | | +|`properties`|A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external data lineage sources can use it to store mapping information.|*Dict[string, string]*| | | | +|[`columns`](./TableYaml.md#columnlineagesourcespecmap)|Configuration of source columns for each column in the current table. The keys in this dictionary are column names in the current table. The object stored in the dictionary contain a list of source columns.|*[ColumnLineageSourceSpecMap](./TableYaml.md#columnlineagesourcespecmap)*| | | | + + + +___ + +## ColumnLineageSourceSpecMap +Dictionary of mapping of source columns to the columns in the current table. + The keys in this dictionary are the column names in the current table. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|`self`||*Dict[string, [ColumnLineageSourceSpec](./TableYaml.md#columnlineagesourcespec)]*| | | | + + + +___ + +## ColumnLineageSourceSpec +Describes the list of source columns for a column in the current table. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|[`source_columns`](./TableYaml.md#sourcecolumnssetspec)|A list of source columns from the source table name from which this column receives data.|*[SourceColumnsSetSpec](./TableYaml.md#sourcecolumnssetspec)*| | | | +|`properties`|A dictionary of mapping properties stored as a key/value dictionary. Data lineage synchronization tools that are importing data lineage mappings from external data lineage sources can use it to store mapping information.|*Dict[string, string]*| | | | + + + +___ + +## SourceColumnsSetSpec +A collection of unique names of source columns from which the current column receives data. This information is used to track column-level data lineage. + + + ___ diff --git a/docs/reference/yaml/monitoring/table-daily-monitoring-checks.md b/docs/reference/yaml/monitoring/table-daily-monitoring-checks.md index afc6ab97db..33f33cd951 100644 --- a/docs/reference/yaml/monitoring/table-daily-monitoring-checks.md +++ b/docs/reference/yaml/monitoring/table-daily-monitoring-checks.md @@ -19,6 +19,7 @@ The structure of this object is described below |[`custom_sql`](./table-daily-monitoring-checks.md#tablecustomsqldailymonitoringchecksspec)|Daily monitoring custom SQL checks|*[TableCustomSqlDailyMonitoringChecksSpec](./table-daily-monitoring-checks.md#tablecustomsqldailymonitoringchecksspec)*| | | | |[`availability`](./table-daily-monitoring-checks.md#tableavailabilitydailymonitoringchecksspec)|Daily monitoring table availability checks|*[TableAvailabilityDailyMonitoringChecksSpec](./table-daily-monitoring-checks.md#tableavailabilitydailymonitoringchecksspec)*| | | | |[`schema`](./table-daily-monitoring-checks.md#tableschemadailymonitoringchecksspec)|Daily monitoring table schema checks|*[TableSchemaDailyMonitoringChecksSpec](./table-daily-monitoring-checks.md#tableschemadailymonitoringchecksspec)*| | | | +|[`uniqueness`](./table-daily-monitoring-checks.md#tableuniquenessdailymonitoringchecksspec)|Daily monitoring uniqueness checks on a table level.|*[TableUniquenessDailyMonitoringChecksSpec](./table-daily-monitoring-checks.md#tableuniquenessdailymonitoringchecksspec)*| | | | |[`comparisons`](./table-daily-monitoring-checks.md#tablecomparisondailymonitoringchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonDailyMonitoringChecksSpecMap](./table-daily-monitoring-checks.md#tablecomparisondailymonitoringchecksspecmap)*| | | | |[`custom`](../profiling/table-profiling-checks.md#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](../profiling/table-profiling-checks.md#customcheckspecmap)*| | | | @@ -129,6 +130,22 @@ The structure of this object is described below +___ + +## TableUniquenessDailyMonitoringChecksSpec +Container of table level daily monitoring for uniqueness data quality checks. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|[`daily_duplicate_record_count`](../../../checks/table/uniqueness/duplicate-record-count.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|*[TableDuplicateRecordCountCheckSpec](../../../checks/table/uniqueness/duplicate-record-count.md)*| | | | +|[`daily_duplicate_record_percent`](../../../checks/table/uniqueness/duplicate-record-percent.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|*[TableDuplicateRecordPercentCheckSpec](../../../checks/table/uniqueness/duplicate-record-percent.md)*| | | | +|[`custom_checks`](../profiling/table-profiling-checks.md#customcategorycheckspecmap)|Dictionary of additional custom checks within this category. The keys are check names defined in the definition section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom check.|*[CustomCategoryCheckSpecMap](../profiling/table-profiling-checks.md#customcategorycheckspecmap)*| | | | + + + ___ ## TableComparisonDailyMonitoringChecksSpecMap diff --git a/docs/reference/yaml/monitoring/table-monthly-monitoring-checks.md b/docs/reference/yaml/monitoring/table-monthly-monitoring-checks.md index 02e044e9ec..9d697b15fc 100644 --- a/docs/reference/yaml/monitoring/table-monthly-monitoring-checks.md +++ b/docs/reference/yaml/monitoring/table-monthly-monitoring-checks.md @@ -19,6 +19,7 @@ The structure of this object is described below |[`custom_sql`](./table-monthly-monitoring-checks.md#tablecustomsqlmonthlymonitoringchecksspec)|Monthly monitoring of custom SQL checks|*[TableCustomSqlMonthlyMonitoringChecksSpec](./table-monthly-monitoring-checks.md#tablecustomsqlmonthlymonitoringchecksspec)*| | | | |[`availability`](./table-monthly-monitoring-checks.md#tableavailabilitymonthlymonitoringchecksspec)|Daily partitioned availability checks|*[TableAvailabilityMonthlyMonitoringChecksSpec](./table-monthly-monitoring-checks.md#tableavailabilitymonthlymonitoringchecksspec)*| | | | |[`schema`](./table-monthly-monitoring-checks.md#tableschemamonthlymonitoringchecksspec)|Monthly monitoring table schema checks|*[TableSchemaMonthlyMonitoringChecksSpec](./table-monthly-monitoring-checks.md#tableschemamonthlymonitoringchecksspec)*| | | | +|[`uniqueness`](./table-monthly-monitoring-checks.md#tableuniquenessmonthlymonitoringchecksspec)|Monthly monitoring uniqueness checks on a table level.|*[TableUniquenessMonthlyMonitoringChecksSpec](./table-monthly-monitoring-checks.md#tableuniquenessmonthlymonitoringchecksspec)*| | | | |[`comparisons`](./table-monthly-monitoring-checks.md#tablecomparisonmonthlymonitoringchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonMonthlyMonitoringChecksSpecMap](./table-monthly-monitoring-checks.md#tablecomparisonmonthlymonitoringchecksspecmap)*| | | | |[`custom`](../profiling/table-profiling-checks.md#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](../profiling/table-profiling-checks.md#customcheckspecmap)*| | | | @@ -124,6 +125,22 @@ The structure of this object is described below +___ + +## TableUniquenessMonthlyMonitoringChecksSpec +Container of table level monthly monitoring for uniqueness data quality checks + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|[`monthly_duplicate_record_count`](../../../checks/table/uniqueness/duplicate-record-count.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|*[TableDuplicateRecordCountCheckSpec](../../../checks/table/uniqueness/duplicate-record-count.md)*| | | | +|[`monthly_duplicate_record_percent`](../../../checks/table/uniqueness/duplicate-record-percent.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|*[TableDuplicateRecordPercentCheckSpec](../../../checks/table/uniqueness/duplicate-record-percent.md)*| | | | +|[`custom_checks`](../profiling/table-profiling-checks.md#customcategorycheckspecmap)|Dictionary of additional custom checks within this category. The keys are check names defined in the definition section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom check.|*[CustomCategoryCheckSpecMap](../profiling/table-profiling-checks.md#customcategorycheckspecmap)*| | | | + + + ___ ## TableComparisonMonthlyMonitoringChecksSpecMap diff --git a/docs/reference/yaml/partitioned/table-daily-partitioned-checks.md b/docs/reference/yaml/partitioned/table-daily-partitioned-checks.md index 740b4fefcd..dda1304978 100644 --- a/docs/reference/yaml/partitioned/table-daily-partitioned-checks.md +++ b/docs/reference/yaml/partitioned/table-daily-partitioned-checks.md @@ -16,6 +16,7 @@ The structure of this object is described below |[`volume`](./table-daily-partitioned-checks.md#tablevolumedailypartitionedchecksspec)|Volume daily partitioned data quality checks that verify the quality of every day of data separately|*[TableVolumeDailyPartitionedChecksSpec](./table-daily-partitioned-checks.md#tablevolumedailypartitionedchecksspec)*| | | | |[`timeliness`](./table-daily-partitioned-checks.md#tabletimelinessdailypartitionedchecksspec)|Daily partitioned timeliness checks|*[TableTimelinessDailyPartitionedChecksSpec](./table-daily-partitioned-checks.md#tabletimelinessdailypartitionedchecksspec)*| | | | |[`custom_sql`](./table-daily-partitioned-checks.md#tablecustomsqldailypartitionedchecksspec)|Custom SQL daily partitioned data quality checks that verify the quality of every day of data separately|*[TableCustomSqlDailyPartitionedChecksSpec](./table-daily-partitioned-checks.md#tablecustomsqldailypartitionedchecksspec)*| | | | +|[`uniqueness`](./table-daily-partitioned-checks.md#tableuniquenessdailypartitionchecksspec)|Daily partitioned uniqueness checks on a table level.|*[TableUniquenessDailyPartitionChecksSpec](./table-daily-partitioned-checks.md#tableuniquenessdailypartitionchecksspec)*| | | | |[`comparisons`](./table-daily-partitioned-checks.md#tablecomparisondailypartitionedchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonDailyPartitionedChecksSpecMap](./table-daily-partitioned-checks.md#tablecomparisondailypartitionedchecksspecmap)*| | | | |[`custom`](../profiling/table-profiling-checks.md#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](../profiling/table-profiling-checks.md#customcheckspecmap)*| | | | @@ -137,6 +138,22 @@ The structure of this object is described below +___ + +## TableUniquenessDailyPartitionChecksSpec +Container of table level daily partition for uniqueness data quality checks. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|[`daily_partition_duplicate_record_count`](../../../checks/table/uniqueness/duplicate-record-count.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|*[TableDuplicateRecordCountCheckSpec](../../../checks/table/uniqueness/duplicate-record-count.md)*| | | | +|[`daily_partition_duplicate_record_percent`](../../../checks/table/uniqueness/duplicate-record-percent.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|*[TableDuplicateRecordPercentCheckSpec](../../../checks/table/uniqueness/duplicate-record-percent.md)*| | | | +|[`custom_checks`](../profiling/table-profiling-checks.md#customcategorycheckspecmap)|Dictionary of additional custom checks within this category. The keys are check names defined in the definition section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom check.|*[CustomCategoryCheckSpecMap](../profiling/table-profiling-checks.md#customcategorycheckspecmap)*| | | | + + + ___ ## TableComparisonDailyPartitionedChecksSpecMap diff --git a/docs/reference/yaml/partitioned/table-monthly-partitioned-checks.md b/docs/reference/yaml/partitioned/table-monthly-partitioned-checks.md index 8790651e26..7bdd6a1fb5 100644 --- a/docs/reference/yaml/partitioned/table-monthly-partitioned-checks.md +++ b/docs/reference/yaml/partitioned/table-monthly-partitioned-checks.md @@ -16,6 +16,7 @@ The structure of this object is described below |[`volume`](./table-monthly-partitioned-checks.md#tablevolumemonthlypartitionedchecksspec)|Volume monthly partitioned data quality checks that verify the quality of every month of data separately|*[TableVolumeMonthlyPartitionedChecksSpec](./table-monthly-partitioned-checks.md#tablevolumemonthlypartitionedchecksspec)*| | | | |[`timeliness`](./table-monthly-partitioned-checks.md#tabletimelinessmonthlypartitionedchecksspec)|Monthly partitioned timeliness checks|*[TableTimelinessMonthlyPartitionedChecksSpec](./table-monthly-partitioned-checks.md#tabletimelinessmonthlypartitionedchecksspec)*| | | | |[`custom_sql`](./table-monthly-partitioned-checks.md#tablecustomsqlmonthlypartitionedchecksspec)|Custom SQL monthly partitioned data quality checks that verify the quality of every month of data separately|*[TableCustomSqlMonthlyPartitionedChecksSpec](./table-monthly-partitioned-checks.md#tablecustomsqlmonthlypartitionedchecksspec)*| | | | +|[`uniqueness`](./table-monthly-partitioned-checks.md#tableuniquenessmonthlypartitionchecksspec)|Monthly partitioned uniqueness checks on a table level.|*[TableUniquenessMonthlyPartitionChecksSpec](./table-monthly-partitioned-checks.md#tableuniquenessmonthlypartitionchecksspec)*| | | | |[`comparisons`](./table-monthly-partitioned-checks.md#tablecomparisonmonthlypartitionedchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonMonthlyPartitionedChecksSpecMap](./table-monthly-partitioned-checks.md#tablecomparisonmonthlypartitionedchecksspecmap)*| | | | |[`custom`](../profiling/table-profiling-checks.md#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](../profiling/table-profiling-checks.md#customcheckspecmap)*| | | | @@ -71,6 +72,22 @@ The structure of this object is described below +___ + +## TableUniquenessMonthlyPartitionChecksSpec +Container of table level monthly partition for uniqueness data quality checks + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|[`monthly_partition_duplicate_record_count`](../../../checks/table/uniqueness/duplicate-record-count.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|*[TableDuplicateRecordCountCheckSpec](../../../checks/table/uniqueness/duplicate-record-count.md)*| | | | +|[`monthly_partition_duplicate_record_percent`](../../../checks/table/uniqueness/duplicate-record-percent.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|*[TableDuplicateRecordPercentCheckSpec](../../../checks/table/uniqueness/duplicate-record-percent.md)*| | | | +|[`custom_checks`](../profiling/table-profiling-checks.md#customcategorycheckspecmap)|Dictionary of additional custom checks within this category. The keys are check names defined in the definition section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom check.|*[CustomCategoryCheckSpecMap](../profiling/table-profiling-checks.md#customcategorycheckspecmap)*| | | | + + + ___ ## TableComparisonMonthlyPartitionedChecksSpecMap diff --git a/docs/reference/yaml/profiling/table-profiling-checks.md b/docs/reference/yaml/profiling/table-profiling-checks.md index 80d21834e7..209303c350 100644 --- a/docs/reference/yaml/profiling/table-profiling-checks.md +++ b/docs/reference/yaml/profiling/table-profiling-checks.md @@ -20,6 +20,7 @@ The structure of this object is described below |[`custom_sql`](./table-profiling-checks.md#tablecustomsqlprofilingchecksspec)|Configuration of data quality checks that are evaluating custom SQL conditions and aggregated expressions.|*[TableCustomSqlProfilingChecksSpec](./table-profiling-checks.md#tablecustomsqlprofilingchecksspec)*| | | | |[`availability`](./table-profiling-checks.md#tableavailabilityprofilingchecksspec)|Configuration of the table availability data quality checks on a table level.|*[TableAvailabilityProfilingChecksSpec](./table-profiling-checks.md#tableavailabilityprofilingchecksspec)*| | | | |[`schema`](./table-profiling-checks.md#tableschemaprofilingchecksspec)|Configuration of schema (column count and schema) data quality checks on a table level.|*[TableSchemaProfilingChecksSpec](./table-profiling-checks.md#tableschemaprofilingchecksspec)*| | | | +|[`uniqueness`](./table-profiling-checks.md#tableuniquenessprofilingchecksspec)|Configuration of uniqueness checks on a table level.|*[TableUniquenessProfilingChecksSpec](./table-profiling-checks.md#tableuniquenessprofilingchecksspec)*| | | | |[`comparisons`](./table-profiling-checks.md#tablecomparisonprofilingchecksspecmap)|Dictionary of configuration of checks for table comparisons. The key that identifies each comparison must match the name of a data comparison that is configured on the parent table.|*[TableComparisonProfilingChecksSpecMap](./table-profiling-checks.md#tablecomparisonprofilingchecksspecmap)*| | | | |[`custom`](./table-profiling-checks.md#customcheckspecmap)|Dictionary of custom checks. The keys are check names within this category.|*[CustomCheckSpecMap](./table-profiling-checks.md#customcheckspecmap)*| | | | @@ -137,6 +138,22 @@ The structure of this object is described below +___ + +## TableUniquenessProfilingChecksSpec +Container of built-in preconfigured uniqueness data quality checks on a table level. + + +The structure of this object is described below + +| Property name | Description                     | Data type | Enum values | Default value | Sample values | +|---------------|---------------------------------|-----------|-------------|---------------|---------------| +|[`profile_duplicate_record_count`](../../../checks/table/uniqueness/duplicate-record-count.md)|Verifies that the number of duplicate record values in a table does not exceed the maximum accepted count.|*[TableDuplicateRecordCountCheckSpec](../../../checks/table/uniqueness/duplicate-record-count.md)*| | | | +|[`profile_duplicate_record_percent`](../../../checks/table/uniqueness/duplicate-record-percent.md)|Verifies that the percentage of duplicate record values in a table does not exceed the maximum accepted percentage.|*[TableDuplicateRecordPercentCheckSpec](../../../checks/table/uniqueness/duplicate-record-percent.md)*| | | | +|[`custom_checks`](./table-profiling-checks.md#customcategorycheckspecmap)|Dictionary of additional custom checks within this category. The keys are check names defined in the definition section. The sensor parameters and rules should match the type of the configured sensor and rule for the custom check.|*[CustomCategoryCheckSpecMap](./table-profiling-checks.md#customcategorycheckspecmap)*| | | | + + + ___ ## TableComparisonProfilingChecksSpecMap diff --git a/docs/working-with-dqo/collecting-basic-data-statistics.md b/docs/working-with-dqo/collecting-basic-data-statistics.md index 7304b04a86..1503bb1533 100644 --- a/docs/working-with-dqo/collecting-basic-data-statistics.md +++ b/docs/working-with-dqo/collecting-basic-data-statistics.md @@ -69,7 +69,7 @@ In the **Table Statistics** box: In the summary table: -- **Dimensions**: The circles represent the results from data quality checks for at least three dimensions: Completeness, Validity, and Consistency. The color of the circle represents the results: white for no checks were run, green for valid results, orange for error results, red for fatal error results, and grey for check execution errors. Hovering over a circle will display more details. +- **Dimensions**: The circles represent the results from data quality checks for at least three dimensions: Completeness, Validity, and Consistency. The color of the circle represents the results: white for no checks were run, green for correct results, orange for error results, red for fatal error results, and grey for check execution errors. Hovering over a circle will display more details. - **Column name**: The name of the column in the selected table. Clicking on the column name will lead you to the detailed statistics. - **Detected data type**: The data type detected for STRING columns, which can be INTEGER, FLOAT, DATETIME, TIMESTAMP, BOOLEAN, STRING, or Mixed data types. - **Imported data type**: The physical data type retrieved from the database metadata, specific to the data source. diff --git a/docs/working-with-dqo/daily-monitoring-of-data-quality.md b/docs/working-with-dqo/daily-monitoring-of-data-quality.md index 0fd63a8dfe..cc94adf6fe 100644 --- a/docs/working-with-dqo/daily-monitoring-of-data-quality.md +++ b/docs/working-with-dqo/daily-monitoring-of-data-quality.md @@ -31,11 +31,24 @@ To import source schemas and tables: ![Importing tables](https://dqops.com/docs/images/working-with-dqo/daily-monitoring-of-data-quality/importing-tables.png){ loading=lazy; width="1200px" } -When new tables are imported, DQOps automatically activates profiling and monitoring checks, and opens an Advisor -that allows you to quickly collect basic statistics, run profiling checks, or modify the schedule for newly imported tables. -But let's close the Advisor with the X button in the top right corner and verify the activation of the default checks. +Upon import, you will receive information that a new tables have been imported. You can then begin collecting basic statistics +and profiling data by running default data profiling checks. Simply click on the **Start profiling** button to initiate this process. -![Importing tables - advisor](https://dqops.com/docs/images/working-with-dqo/daily-monitoring-of-data-quality/advisor.png){ loading=lazy; width="1200px" } +![Collect basic statistics and profile data with default profiling checks](https://dqops.com/docs/images/getting-started/collect-basic-statistics-and-profile-data.png) + +!!! info "Automatically activated checks" + + Once new tables are imported, DQOps automatically activates [profiling and monitoring checks](../dqo-concepts/definition-of-data-quality-checks/index.md). + These checks include row count, table availability, and checks detecting schema changes. The profiling checks are scheduled + to run at 1:00 a.m. on the 1st day of every month, and the monitoring checks are scheduled to run daily at 12:00 p.m. + + [**Profiling checks**](../dqo-concepts/definition-of-data-quality-checks/data-profiling-checks.md) are designed to assess + the initial data quality score of a data source. Profiling checks are also useful for exploring and experimenting with + various types of checks and determining the most suitable ones for regular data quality monitoring. + + [**Monitoring checks**](../dqo-concepts/definition-of-data-quality-checks/data-observability-monitoring-checks.md) are + standard checks that monitor the data quality of a table or column. They can also be referred to as **Data Observability** checks. + These checks capture a single data quality result for the entire table or column. ## Verify activation of the default checks @@ -82,7 +95,7 @@ On the left of the name of each check, there are several buttons and icons that - View detailed results for Checks, Sensor readouts, and Execution errors - View detailed information about check, - Check the results of the run check shown as a color square - - Green for a valid result + - Green for a correct result - Yellow for a warning - Orange for an error - Red for a fatal error @@ -100,7 +113,7 @@ or table and click on the **Run checks** option. A dialog box will appear where you can modify different options or simply click the **Run checks** button. -![Run all checks dialog box](https://dqops.com/docs/images/working-with-dqo/daily-monitoring-of-data-quality/run-all-checks-dialog-box.png){ loading=lazy; width="1000px" } +![Run all checks dialog box](https://dqops.com/docs/images/working-with-dqo/daily-monitoring-of-data-quality/run-all-checks-dialog-box2.png){ loading=lazy; width="1000px" } diff --git a/docs/working-with-dqo/managing-data-quality-incidents-with-dqops.md b/docs/working-with-dqo/managing-data-quality-incidents-with-dqops.md index 039d43e047..9d8ead39b0 100644 --- a/docs/working-with-dqo/managing-data-quality-incidents-with-dqops.md +++ b/docs/working-with-dqo/managing-data-quality-incidents-with-dqops.md @@ -1,4 +1,4 @@ -# Incidents overview +# Incidents and notifications overview Read this guide to learn how the data quality incident workflow works in DQOps, and how to use all incident management screens to manage data quality incidents. ## Overview @@ -43,13 +43,13 @@ Incidents are the default function of DQOps and automatically groups issues. To modify the settings of the Incidents, follow these steps: -![Incidents and notifications tab](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incidents-and-notifications-settings2.png){ loading=lazy; width="1200px" } +![Incidents and notifications tab](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incident-grouping.png){ loading=lazy; width="1200px" } 1. Go to the **Data Sources** section. 2. Select the relevant data source from the tree view on the left. -3. Select the **Incidents And Notifications** tab. +3. Select the **Incident grouping** tab. 4. Update the settings and click the **Save** button @@ -107,6 +107,7 @@ filter, sort, and view detailed information about the incidents. For each incident the following information is provided: - **Resolution status**: A status assigned to the incident (Open, Acknowledged, Resolved or Muted) +- **Severity**: A colored square indicating the severity level, yellow for warning, orange for error and red for fata error. - **Total issues**: The number of times the data quality issue has occurred in the selected time range - **Schema**: Name of the schema in which data quality issues were detected - **Table**: Name of the table in which data quality issues were detected @@ -178,12 +179,12 @@ filter and sort them. The upper right corner of the Incident details screen provides access to several helpful actions: -![Incident details links](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incident-details-links.png) +![Incident details links](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/incident-details-links2.png) -- **Check Editor link**: Clicking this link will take you directly to the **Check editor** for the specific check that triggered the incident. This allows you to examine the check's configuration. - **Disable checks for the incident**: This button allows you to temporarily disable the check responsible for the incident. Disabling a check can be useful when you are actively working to resolve the underlying issue. -- **Recalibrate checks for the incident**: Clicking on this button will decrease the check's thresholds by 30%. +- **Reconfigure rule thresholds**: Clicking on this button will decrease the rule threshold for the data quality check that caused the incident by 30% to reduce the number of data quality issues. - **Change incident configuration**: This button opens the **Incident configuration** screen for the table where the incident originated. This allows you to manage incident settings, such as grouping and severity levels. +- **Configure notification for this incident**: This button allows you to create new or modify existing notification for this incident. ### **Filter data quality check results** @@ -213,13 +214,13 @@ If the check encountered any errors during execution, they will be displayed in ### **Disable check for the incident** -To manage incidents, you have the option to disable the check responsible for the incident. Disabling a check can be +To manage incidents, you have the option to disable the check responsible for the incident. Disabling a check can be useful when you are actively working to resolve the underlying issue. -To disable a check, click on the "Disable check" button in the top right corner of the Incident details screen. +To disable a check, click on the "Disable check" button in the top right corner of the Incident details screen. Once confirmed, the check will be temporarily stopped from executing. -![Disabling check](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/disabling-check-button.png){ loading=lazy; width="1200px" } +![Disabling check](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/disabling-check-button2.png){ loading=lazy; width="1200px" } You can verify that the check has been disabled by navigating to the check editor screen. @@ -228,16 +229,16 @@ You can verify that the check has been disabled by navigating to the check edito ### **Recalibrate check for the incident** -DQOps offers a one-click option to automatically reduce the number of data quality issues identified by a check. +DQOps offers a one-click option to automatically reduce the number of data quality issues identified by a check. This can be helpful in situations where the check might be overly sensitive. -Clicking the Recalibrate button will decrease the check's thresholds by 30%. For more significant adjustments, you can click -the Recalibrate button multiple times. Each click will further reduce the check's thresholds by an additional 30%. +Clicking the **Reconfigure** button will decrease the rule threshold for the data quality check that caused the incident by 30%. For more significant adjustments, you can click +the **Recalibrate** button multiple times. Each click will further reduce the check's thresholds by an additional 30%. -![Recalibrate-check-button](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/recalibrate-check-button.png){ loading=lazy; width="1200px" } - -The following example YAML files illustrate the `daily_partition_row_count` check configuration before and after recalibration. Notice that the `min_count` rule has been reduced from 1000, to 700. +![Reconfigure check button](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/recalibrate-check-button2.png){ loading=lazy; width="1200px" } +The following example YAML files illustrate the `daily_partition_row_count` check configuration before and after reconfiguration. +Notice that the `min_count` rule has been reduced from 1000, to 700. === "Check configuration before recalibration" @@ -282,6 +283,34 @@ The following example YAML files illustrate the `daily_partition_row_count` chec error: min_count: 700 ``` +### **Configure notification for an incident** + +To receive notifications for specific data quality incidents, you can create or edit notification filters. +To do this, click on the envelope icon located in the top right corner of the incident details screen. + +![Configure notification filter](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configure-notificaiton-filter.png){ loading=lazy; width="1200px" } + +The action after clicking on the envelope icon will depend on whether the incident matches any existing incident +[notification filters](../dqo-concepts/grouping-data-quality-issues-to-incidents.md#notification-filters). + +**Adding a new notification filter**. + +If no notification has ever been configured for the incident, you will see a popup message informing you to create a new notification configuration. + +![Configure incident popup](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/configure-incident-popup.png){ loading=lazy; width="600px" } + +Once approved, you will be redirected to the page to create the new incident notification filter. +The configuration form will be partially filled based on the incident's data. The details of the filter configuration can be +found in the section [Notification filters](../dqo-concepts/grouping-data-quality-issues-to-incidents.md#notification-filters) below. + +![New notification filter](https://dqops.com/docs/images/working-with-dqo/incidents-and-notifications/new-notification-filter.png){ loading=lazy; width="1200px" } + + +**Editing an existing notification filter** + +If the notification matches an existing filter, a screen with this filter will open, and you can edit the filter's settings. + +Learn more about [configuration of the notification filtering in the Concept section](../dqo-concepts/grouping-data-quality-issues-to-incidents.md#incident-notifications) ## What's next diff --git a/docs/working-with-dqo/run-data-quality-checks.md b/docs/working-with-dqo/run-data-quality-checks.md index fcfb74e795..afcffa676a 100644 --- a/docs/working-with-dqo/run-data-quality-checks.md +++ b/docs/working-with-dqo/run-data-quality-checks.md @@ -20,22 +20,28 @@ To navigate to the Check editor: 1. Click on the **Profiling**, **Monitoring checks** or **Partition checks** section at the top of the screen. - ![Navigate to check section](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/navigate-to-the-check-section2.png){ loading=lazy; width="1200px" } + ![Navigate to check section](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/navigate-to-the-check-section3.png){ loading=lazy; width="1200px" } -2. On the tree view on the left, select a table or column of interest by expanding the connection. +2. On the tree view on the left, select a table or column of interest by expanding the connection and select the **Data quality check editor** tab, + (named **Profiling checks editor** in the Profiling section). This will open a [**Check editor**](../dqo-concepts/dqops-user-interface-overview.md#check-editor) screen when you can work with checks. + If you select a column directly, the check editor will open immediately without requiring additional steps. - ![Select a table or column of interest](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/select-a-table-or-colum-of-interest3.png){ loading=lazy; width="1200px" } + ![Select a table or column of interest](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/select-a-table-or-colum-of-interest4.png){ loading=lazy; width="1200px" } - The Check editor screen has tabs that allow you to switch between Profiling (only in Profiling section) or Daily - and Monthly checks, review Table quality status, access the screen for setting Comparisons, view Basic data statistics - (only in Profiling section), or preview tables (only in Profiling section). +The table with data quality checks contains a list of checks divided into different data quality subcategories that you +can expand and collapse by clicking on an arrow. [Learn more about the different check categories.](../dqo-concepts/definition-of-data-quality-checks/index.md#categories-of-checks) - The table with data quality checks contains a list of checks divided into different data quality subcategories that you - can expand and collapse by clicking on an arrow. [Learn more about the different check categories.](../dqo-concepts/definition-of-data-quality-checks/index.md#categories-of-checks) +By clicking on the **Show advanced checks** checkbox, located at the top of the table, you can view all checks available in DQOps. + +The right side of the table allows selecting different threshold levels (issue severity levels). [Learn more about threshold levels.](../dqo-concepts/definition-of-data-quality-checks/index.md#issue-severity-levels) + +There are two types of checks editors **Simplified** and **Advanced**. The primary difference lies in the ability to define multiple severity levels in the Advanced mode. +To access the Advanced mode, select the **Multiple levels** option from the Issue severity level dropdown menu in the simplified data check editor. +[Learn more about Simplified and Advanced check editor in the concept section](../dqo-concepts/dqops-user-interface-overview.md#check-editor). - The right side of the table allows setting different threshold levels (severity levels). [Learn more about threshold levels.](../dqo-concepts/definition-of-data-quality-checks/index.md#issue-severity-levels) +The Check editor screen in the paid version of DQOps has additional subtabs that allow you to switch between Daily and Monthly checks. ### **Run a check from a check editor** @@ -43,65 +49,48 @@ To run a check: 1. Activate the check of interest by clicking the toggle button next to the check name in the list on the right. - ![Activate check](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/enable-check3.png){ loading=lazy; width="1200px" } + The gray color indicates that the check is not activated, light green indicates [default checks](../dqo-concepts/data-observability.md#default-table-level-checks) that are activated, and darker green indicates manually activated checks. + + ![Activate check](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/enable-check4.png){ loading=lazy; width="1200px" } -2. Set the threshold levels or leave default values. Set parameters if the check has any. Click the **Save** button in the upper right corner. +2. Choose the **Issue severity level** from the dropdown list and set the **Rule thresholds** or leave default values. Set parameters if the check has any. Click the **Save** button in the upper right corner. You can read more about [threshold severity levels in DQOps concepts section](../dqo-concepts/definition-of-data-quality-checks/index.md#issue-severity-levels). - ![Set threshold levels](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/set-threshold-levels3.png){ loading=lazy; width="1200px" } - -3. Run data quality check by clicking the **Run Check** icon - - ![Run check](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/run-check3.png){ loading=lazy; width="1200px" } - - A square should appear next to the name of the checks indicating the results of the run check: + ![Set threshold levels](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/set-threshold-levels4.png){ loading=lazy; width="1200px" } - - Green for a valid result - - Yellow for a warning - - Orange for an error - - Red for a fatal error - - Black for execution error - - You can view the details by placing the mouse cursor on the square. - - ![View quick check results](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/view-quick-check-results1.png){ loading=lazy } - - This check run resulted in a valid result. The daily_row_count sensor readout was 18 155, which was higher than the min_count error threshold 1 000. - -### **Run checks from a tree view** +3. Run data quality check by clicking the **Run check** icon -You can run checks for your entire connection, schema, table, or column from the tree view on the left-hand -side of the screen. There are also additional parameters you can select. + ![Run check](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/run-check4.png){ loading=lazy; width="1200px" } -To do so, click on the three-dot icon and select the **Run check** option. -This will run all the activated checks for the selected connection, schema, table, or column. +A square should appear next to the name of the checks indicating the results of the run check: -![Run checks from the tree view](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/run-checks-from-the-tree-view.png){ loading=lazy; width="1200px" } +- Green for a correct result +- Yellow for a warning +- Orange for an error +- Red for a fatal error +- Black for execution error -A dialog box will appear where you can modify different options such as connection, schema and table name, column date, and column data type. -There are also additional parameters when you can narrow the selection to the check type, check name, sensor name, table -comparison name, labels, or tags. +Hover over the square to view a quick summary, or click the **Results** icon for a detailed results. -After selecting options simply click the **Run checks** button to run checks. +![View quick check results](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/quick-check-results2.png){ loading=lazy } -![Run all checks dialog box](https://dqops.com/docs/images/working-with-dqo/daily-monitoring-of-data-quality/run-all-checks-dialog-box.png){ loading=lazy; width="1000px" } +This check run resulted in an error. The column has 17 155 rows with null values, which was higher than the max_count error threshold 0. +You can also run a check for the entire category by clicking on the **Run** button located to the left of the category name. -If you run **Partition checks**, you will have the additional option to choose from different incremental time windows -when running from the tree view. +![View quick check results](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/run-check-for-the-entire-category.png){ loading=lazy } -![Run partition checks from the tree view](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/run-partition-checks-from-the-tree-view.png){ loading=lazy; width="1200px" } ### **View detailed check results** -To view detailed check results, sensor readouts, and execution errors, click on the **Results** icon. +To view detailed check results, sensor readouts, execution errors, or [error samples](../dqo-concepts/data-quality-error-sampling.md) click on the **Results** icon. -![Checking results](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/detailed-check-results1.png){ loading=lazy; width="1200px" } +![Checking results](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/detailed-check-results2.png){ loading=lazy; width="1200px" } -A table will appear with detailed information about the run check. You can filter the table by data group and month -using the dropdowns. Additionally, you can switch between the table and chart view by clicking on the icons on the -right of the month filter dropdown. To close the detailed results view, click on the **Results** icon again or use +A table will appear with detailed information about the run check. You can filter the table by data group and month +using the dropdowns. Additionally, you can switch between the table and chart view by clicking on the icons on the +right of the month filter dropdown. To close the detailed results view, click on the **Results** icon again or use the X button on the right. [Learn here how to delete data quality results](delete-data-quality-results.md). @@ -112,6 +101,29 @@ To synchronize all the data click on the **Synchronize** button in the upper rig You can learn here how to [Review the results of data quality monitoring on dashboards.](review-the-data-quality-results-on-dashboards.md) +### **Run checks from a tree view** + +You can run checks for your entire connection, schema, table, or column from the tree view on the left-hand +side of the screen. There are also additional parameters you can select. + +To do so, click on the three-dot icon next to the table or column name in the tree view and select the **Run check** option. +This will run all the activated checks for the selected connection, schema, table, or column. + +![Run checks from the tree view](https://dqops.com/docs/images/working-with-dqo/run-data-quality-checks/run-checks-from-the-tree-view2.png){ loading=lazy; width="1200px" } + +A dialog box will appear where you can modify different options such as connection, schema and table name, column date, and column data type. +There are also additional parameters when you can narrow the selection to the check type, check name, sensor name, table +comparison name, labels, or tags. + +After selecting options simply click the **Run checks** button to run checks. + +![Run all checks dialog box](https://dqops.com/docs/images/working-with-dqo/daily-monitoring-of-data-quality/run-all-checks-dialog-box2.png){ loading=lazy; width="1000px" } + +If you want to run **Partition checks**, you can select from various incremental time window options here. + +### **View detailed check results** + +When you select **Partition checks** type, you will have the additional option to select the time window. ## Run data quality checks using the DQOps Shell @@ -138,7 +150,7 @@ To add and run data quality checks using the DQOps Shell, follow the steps below Below is an example of the YAML file showing a sample configuration of a profiling table check row_count. Some columns were truncated for clarity - ```yaml hl_lines="9-13" + ```yaml hl_lines="14-19" apiVersion: dqo/v1 kind: table spec: @@ -147,28 +159,17 @@ To add and run data quality checks using the DQOps Shell, follow the steps below incremental_time_window: daily_partitioning_recent_days: 7 monthly_partitioning_recent_months: 1 - profiling_checks: - volume: - row_count: - error: - min_count: 0 columns: - unique_key: - type_snapshot: - column_type: INT64 - nullable: true - address: + city: type_snapshot: column_type: STRING nullable: true - census_tract: - type_snapshot: - column_type: FLOAT64 - nullable: true - clearance_date: - type_snapshot: - column_type: TIMESTAMP - nullable: true + monitoring_checks: + daily: + nulls: + daily_nulls_count: + error: + max_count: 0 ``` 4. To execute the check, run the following command in DQOps Shell: @@ -184,11 +185,11 @@ To add and run data quality checks using the DQOps Shell, follow the steps below ``` Check evaluation summary per table: - +--------------+------------------+------+--------------+-------------+--------+------+------------+----------------+ - |Connection |Table |Checks|Sensor results|Valid results|Warnings|Errors|Fatal errors|Execution errors| - +--------------+------------------+------+--------------+-------------+--------+------+------------+----------------+ - |testconnection|austin_crime.crime|1 |1 |1 |0 |0 |0 |0 | - +--------------+------------------+------+--------------+-------------+--------+------+------------+----------------+ + +--------------------+-------------------------------+------+--------------+-------------+--------+------+------------+----------------+ + |Connection |Table |Checks|Sensor results|Valid results|Warnings|Errors|Fatal errors|Execution errors| + +--------------------+-------------------------------+------+--------------+-------------+--------+------+------------+----------------+ + |bigquery-public-data|austin_311.311_service_requests|1 |1 |0 |0 |1 |0 |0 | + +--------------------+-------------------------------+------+--------------+-------------+--------+------+------------+----------------+ ``` For a more detailed insight of how the check is run, you can initiate the check in debug mode by executing the @@ -202,15 +203,16 @@ To add and run data quality checks using the DQOps Shell, follow the steps below ``` ************************************************** - Executing SQL on connection testconnection (bigquery) + Executing SQL on connection bigquery-public-data (bigquery) SQL to be executed on the connection: SELECT - COUNT(*) AS actual_value, - CURRENT_TIMESTAMP() AS time_period, - TIMESTAMP(CURRENT_TIMESTAMP()) AS time_period_utc - FROM `bigquery-public-data`.`austin_crime`.`crime` AS analyzed_table - GROUP BY time_period, time_period_utc - ORDER BY time_period, time_period_utc + SUM( + CASE + WHEN analyzed_table.`city` IS NULL THEN 1 + ELSE 0 + END + ) AS actual_value + FROM `bigquery-public-data`.`austin_311`.`311_service_requests` AS analyzed_table ************************************************** ``` @@ -218,18 +220,16 @@ To add and run data quality checks using the DQOps Shell, follow the steps below ``` Results returned by the sensor: - +------------+------------------------+------------------------+ - |actual_value|time_period |time_period_utc | - +------------+------------------------+------------------------+ - |116675 |2023-05-12T13:45:46.602Z|2023-05-12T13:45:46.602Z| - +------------+------------------------+------------------------+ - ************************************************** + +------------+ + |actual_value| + +------------+ + |17544 | + +------------+ ``` ## What's next -- Now you might want to check how to [delete data quality results](delete-data-quality-results.md). +- Now, you might want to learn how DQOps [automatically configures the rule thresholds in data quality checks to find the most common data quality issues](../dqo-concepts/data-quality-rule-mining.md) +- Learn how to [delete data quality results](delete-data-quality-results.md). - With DQOps you can [activate, deactivate or modify multiple data quality checks at once](activate-and-deactivate-multiple-checks.md). Follow the link to learn more. -- DQOps provide you with summary statistics about your table and column. This information can be valuable in deciding which data quality checks and threshold levels should be set to monitor data quality. For more details about [Basic data statistics, click here](collecting-basic-data-statistics.md). -- Learn more how [targeting data quality checks](../dqo-concepts/running-data-quality-checks.md) when you want to run - data quality checks only for selected tables or columns, using filters. +- DQOps provide you with summary statistics about your table and column. This information can be valuable in deciding which data quality checks and threshold levels should be set to monitor data quality. For more details about [Basic data statistics, click here](collecting-basic-data-statistics.md). \ No newline at end of file diff --git a/dqops/pom.xml b/dqops/pom.xml index 5cabca98c2..ccf3fdf3bb 100644 --- a/dqops/pom.xml +++ b/dqops/pom.xml @@ -224,6 +224,12 @@ com.google.cloud google-cloud-bigquery 2.31.1 + + + commons-logging + commons-logging + + com.google.cloud @@ -234,6 +240,12 @@ com.google.cloud google-cloud-secretmanager 2.23.0 + + + commons-logging + commons-logging + + tech.tablesaw @@ -283,6 +295,10 @@ org.apache.commons commons-text + + commons-logging + commons-logging + @@ -645,6 +661,10 @@ org.apache.hive hive-llap-server + + commons-logging + commons-logging + diff --git a/dqops/sampledata/detect_datatype_test.csv b/dqops/sampledata/detect_datatype_test.csv index c82de391f9..d1a6e8b22f 100644 --- a/dqops/sampledata/detect_datatype_test.csv +++ b/dqops/sampledata/detect_datatype_test.csv @@ -1,9 +1,10 @@ -int1:STRING,float2:STRING,date3:STRING,datetime4:STRING,timestamp5:STRING,bool6:STRING,string7:STRING,mixed8:STRING,date:LOCAL_DATE -1,0.2441,19/02/2020,10/12/2020 0:00:01,2015-10-28 T 10:45:00 UTC,TRUE,String,2020/02/19,2020-01-01 -2,-2.93759,19-02-2020,10-12-2020 12:00:01 AM,2020-01-31T03:51:22Z,FALSE,Napis,0.192,2020-02-01 -3465,0.2345,19.02.2020,10.12.2020 0:00:01 AM,2020-01-31T03:51:22.123456+03,T,Line,10,2020-03-01 -9302,-1.345,2020/02/19,2020/12/10 0:00:01,2020-01-31 T 03:51:22-07,F,New line,2020-12-10 0:00:01,2020-04-01 -12438,4.5789,2020-02-19,2020-12-10 0:00:01,2020-01-31T03:51:22+03:00,Y,This is String,something,2020-05-01 -12,349.12094,2020.02.19,2020.12.10 0:00:01,2020-01-31T03:51:22GMT+03:00,N,Is this a String?,TRUE,2020-06-01 --5,1,,,2020-01-31T03:51:22+0300,yes,,2020-01-31T03:51:22Z,2020-06-02 --123456,-2,,,2020-01-31T03:51:22.123+0300,no,,abc-001,2020-06-03 \ No newline at end of file +id:INTEGER,int1:STRING,float2:STRING,date3:STRING,datetime4:STRING,timestamp5:STRING,bool6:STRING,string7:STRING,mixed8:STRING,date:LOCAL_DATE +1,1,0.2441,19/02/2020,10/12/2020 0:00:01,2015-10-28 T 10:45:00 UTC,TRUE,String,2020/02/19,2020-01-01 +2,2,-2.93759,19-02-2020,10-12-2020 12:00:01 AM,2020-01-31T03:51:22Z,FALSE,Napis,0.192,2020-02-01 +3,3465,0.2345,19.02.2020,10.12.2020 0:00:01 AM,2020-01-31T03:51:22.123456+03,T,Line,10,2020-03-01 +4,9302,-1.345,2020/02/19,2020/12/10 0:00:01,2020-01-31 T 03:51:22-07,F,New line,2020-12-10 0:00:01,2020-04-01 +5,12438,4.5789,2020-02-19,2020-12-10 0:00:01,2020-01-31T03:51:22+03:00,Y,This is String,something,2020-05-01 +6,12,349.12094,2020.02.19,2020.12.10 0:00:01,2020-01-31T03:51:22GMT+03:00,N,Is this a String?,TRUE,2020-06-01 +7,-5,1,,,2020-01-31T03:51:22+0300,yes,,2020-01-31T03:51:22Z,2020-06-02 +8,-123456,-2,,,2020-01-31T03:51:22.123+0300,no,,abc-001,2020-06-03 +9,,,,,,,,,2020-06-03 \ No newline at end of file diff --git a/dqops/sampledata/files/csv/thelook-ecommerce/orders.csv b/dqops/sampledata/files/csv/thelook-ecommerce/orders.csv new file mode 100644 index 0000000000..8398f8a3b5 --- /dev/null +++ b/dqops/sampledata/files/csv/thelook-ecommerce/orders.csv @@ -0,0 +1,1001 @@ +order_id,user_id,status,gender,created_at,returned_at,shipped_at,delivered_at,num_of_item +1,3,Shipped,F,8/12/2019 15:15:00,,8/15/2019 14:08:00,,1 +2,5,Shipped,M,1/20/2022 9:17:00,,1/22/2022 2:25:00,,1 +3,6,Processing,F,7/23/2022 11:33:00,,,,4 +4,6,Complete,F,3/1/2020 11:33:00,,3/2/2020 15:57:00,3/4/2020 16:32:00,2 +5,7,Complete,F,5/7/2023 14:55:00,,5/9/2023 16:21:00,5/10/2023 18:18:00,1 +6,8,Processing,M,6/20/2024 10:48:00,,,,1 +7,8,Complete,M,8/28/2022 10:48:00,,8/30/2022 19:28:00,8/31/2022 0:56:00,2 +8,9,Returned,F,1/7/2022 1:17:00,1/11/2022 1:20:00,1/7/2022 1:57:00,1/9/2022 9:34:00,1 +9,9,Shipped,F,2/13/2023 1:17:00,,2/14/2023 13:24:00,,1 +10,10,Shipped,M,9/24/2022 2:46:00,,9/25/2022 4:14:00,,1 +11,11,Shipped,F,10/14/2020 10:33:00,,10/15/2020 8:39:00,,1 +12,13,Complete,M,12/22/2023 3:05:00,,12/23/2023 7:52:00,12/23/2023 12:19:00,1 +13,13,Shipped,M,3/29/2021 3:05:00,,3/31/2021 1:02:00,,1 +14,13,Shipped,M,4/16/2023 3:05:00,,4/18/2023 15:10:00,,2 +15,13,Complete,M,8/9/2023 3:05:00,,8/9/2023 7:27:00,8/11/2023 4:53:00,2 +16,14,C,F,4/23/2024 8:28:00,,4/24/2024 4:10:00,4/28/2024 14:11:00,1 +17,15,Processing,M,9/7/2024 0:02:37,,,,1 +18,17,Shipped,F,10/31/2022 3:12:00,,11/1/2022 18:38:00,,1 +19,18,Returned,M,3/26/2023 0:11:00,4/2/2023 8:54:00,3/28/2023 19:10:00,4/2/2023 3:24:00,1 +20,19,Returned,M,9/6/2024 9:20:37,9/12/2024 1:59:37,9/9/2024 7:51:37,9/10/2024 18:02:37,1 +21,19,Shipped,M,9/7/2024 9:20:37,,9/8/2024 21:08:37,,2 +22,20,Complete,M,7/27/2023 12:19:00,,7/29/2023 9:54:00,7/31/2023 21:31:00,1 +23,20,Cancelled,M,2/2/2024 12:19:00,,,,2 +24,20,Shipped,M,8/9/2023 12:19:00,,8/10/2023 2:31:00,,1 +25,21,Shipped,F,4/7/2023 5:02:00,,4/9/2023 10:13:00,,2 +26,22,Processing,F,1/4/2021 0:31:00,,,,1 +27,23,Cancelled,M,8/25/2021 6:27:00,,,,2 +28,23,Shipped,M,3/3/2023 6:27:00,,3/5/2023 10:41:00,,1 +29,23,Shipped,M,2/4/2023 6:27:00,,2/5/2023 2:47:00,,1 +30,24,Processing,M,4/28/2024 13:06:00,,,,1 +31,25,Processing,F,5/18/2023 7:09:00,,,,2 +32,26,Returned,F,6/8/2022 3:15:00,6/16/2022 1:05:00,6/11/2022 0:42:00,6/13/2022 14:14:00,1 +33,27,Processing,F,9/27/2022 1:24:00,,,,1 +34,27,Shipped,F,4/30/2021 1:24:00,,5/2/2021 7:41:00,,3 +35,28,Complete,M,9/9/2024 14:38:38,,9/10/2024 21:16:38,9/12/2024 20:02:38,1 +36,29,Shipped,F,9/9/2024 2:39:00,,9/11/2024 12:57:00,,1 +37,29,Complete,F,11/25/2022 2:39:00,,11/25/2022 23:20:00,11/29/2022 20:20:00,3 +38,31,Shipped,M,1/31/2024 11:46:00,,2/3/2024 7:26:00,,1 +39,32,Shipped,M,2/6/2023 1:12:00,,2/8/2023 3:39:00,,1 +40,33,Shipped,F,9/10/2024 3:01:38,,9/11/2024 22:02:38,,4 +41,33,Returned,F,9/10/2024 3:01:38,9/14/2024 7:36:38,9/12/2024 17:12:38,9/13/2024 0:23:38,1 +42,34,Complete,M,12/18/2022 11:44:00,,12/21/2022 8:49:00,12/22/2022 8:41:00,1 +43,35,Cancelled,F,12/23/2023 11:26:00,,,,1 +44,36,Shipped,M,2/7/2024 3:07:00,,2/8/2024 6:06:00,,1 +45,37,Cancelled,M,7/1/2023 12:09:00,,,,2 +46,38,Returned,M,1/19/2024 17:17:00,1/26/2024 4:21:00,1/21/2024 12:15:00,1/26/2024 2:21:00,1 +47,39,Shipped,F,6/15/2021 18:52:00,,6/17/2021 13:12:00,,1 +48,39,Shipped,F,2/27/2022 18:52:00,,3/1/2022 22:08:00,,2 +49,39,Complete,F,9/17/2023 18:52:00,,9/20/2023 17:39:00,9/22/2023 23:22:00,1 +50,39,Cancelled,F,11/21/2021 18:52:00,,,,2 +51,40,Shipped,M,6/5/2021 12:38:00,,6/5/2021 15:09:00,,1 +52,40,Cancelled,M,2/2/2023 12:38:00,,,,3 +53,42,Complete,M,2/7/2024 1:25:00,,2/7/2024 3:35:00,2/8/2024 5:29:00,1 +54,43,Cancelled,M,11/8/2023 2:25:00,,,,1 +55,45,Shipped,M,6/26/2022 18:58:00,,6/27/2022 2:24:00,,1 +56,46,Complete,F,8/18/2024 4:52:00,,8/21/2024 2:16:00,8/23/2024 8:03:00,1 +57,46,Processing,F,8/22/2024 4:52:00,,,,1 +58,47,Complete,M,6/11/2024 7:01:00,,6/13/2024 6:26:00,6/17/2024 9:41:00,2 +59,48,Proc,M,10/14/2019 1:34:00,,,,2 +60,48,Shipped,M,9/18/2021 1:34:00,,9/18/2021 7:22:00,,1 +61,48,Complete,M,3/20/2024 1:34:00,,3/20/2024 17:26:00,3/21/2024 1:18:00,2 +62,49,Shipped,F,2/19/2023 9:19:00,,2/20/2023 4:22:00,,1 +63,49,Shipped,F,12/22/2022 9:19:00,,12/24/2022 5:35:00,,1 +64,51,Returned,F,4/30/2020 16:18:00,5/8/2020 8:39:00,5/3/2020 1:10:00,5/6/2020 13:59:00,1 +65,51,Complete,F,12/21/2022 16:18:00,,12/22/2022 20:39:00,12/24/2022 6:13:00,1 +66,51,Complete,F,5/29/2024 16:18:00,,6/1/2024 12:18:00,6/4/2024 16:50:00,1 +67,52,Complete,M,11/12/2023 8:53:00,,11/14/2023 8:25:00,11/14/2023 12:22:00,1 +68,53,Complete,M,2/12/2024 8:35:00,,2/13/2024 16:54:00,2/14/2024 2:23:00,1 +69,53,Shipped,M,7/9/2024 8:35:00,,7/10/2024 15:36:00,,1 +70,55,Complete,M,4/1/2024 16:08:00,,4/2/2024 0:30:00,4/2/2024 7:18:00,2 +71,55,Complete,M,8/23/2024 16:08:00,,8/25/2024 7:42:00,8/29/2024 17:05:00,3 +72,55,Shipped,M,11/20/2022 16:08:00,,11/23/2022 9:58:00,,1 +73,56,Shipped,M,1/14/2022 13:23:00,,1/15/2022 10:56:00,,2 +74,58,Cancelled,M,6/28/2024 7:30:00,,,,1 +75,58,Cancelled,M,12/27/2021 7:30:00,,,,1 +76,60,Processing,F,6/15/2020 3:57:00,,,,1 +77,61,Complete,F,9/11/2024 17:29:38,,9/12/2024 1:35:38,9/15/2024 4:39:38,1 +78,63,Shipped,F,6/24/2024 16:28:00,,6/26/2024 17:30:00,,1 +79,63,Processing,F,5/29/2023 16:28:00,,,,1 +80,63,Cancelled,F,1/11/2024 16:28:00,,,,3 +81,63,Cancelled,F,1/14/2024 16:28:00,,,,1 +82,64,Shipped,F,9/10/2024 11:44:38,,9/11/2024 0:28:38,,1 +83,66,Shipped,F,6/19/2024 6:19:00,,6/21/2024 15:35:00,,1 +84,67,Cancelled,F,3/23/2023 3:39:00,,,,1 +85,67,Returned,F,7/25/2023 3:39:00,7/26/2023 3:24:00,7/25/2023 15:12:00,7/26/2023 3:12:00,2 +86,69,Shipped,F,4/5/2023 18:10:00,,4/6/2023 18:25:00,,1 +87,70,Complete,M,8/18/2023 5:21:00,,8/19/2023 8:16:00,8/22/2023 4:09:00,1 +88,70,Shipped,M,12/22/2021 5:21:00,,12/24/2021 13:43:00,,1 +89,70,Complete,M,2/5/2020 5:21:00,,2/5/2020 9:24:00,2/9/2020 0:07:00,2 +90,70,Returned,M,3/15/2020 5:21:00,3/20/2020 22:35:00,3/17/2020 16:19:00,3/20/2020 9:57:00,4 +91,71,Cancelled,M,12/12/2021 9:07:00,,,,1 +92,72,Cancelled,F,6/27/2024 4:47:00,,,,1 +93,73,Processing,F,3/27/2020 17:16:00,,,,1 +94,75,Shipped,F,9/4/2024 5:47:00,,9/4/2024 8:55:00,,2 +95,75,Processing,F,9/9/2024 5:47:00,,,,1 +96,75,Shipped,F,8/30/2024 5:47:00,,8/30/2024 18:45:00,,1 +97,75,Shipped,F,9/7/2024 5:47:00,,9/9/2024 4:56:00,,1 +98,76,Cancelled,F,9/16/2020 13:12:00,,,,1 +99,77,Complete,M,6/22/2022 18:25:00,,6/24/2022 5:37:00,6/27/2022 10:55:00,2 +100,78,Shipped,M,6/10/2024 5:55:00,,6/10/2024 11:11:00,,1 +101,79,Shipped,M,2/19/2024 2:14:00,,2/21/2024 20:40:00,,4 +102,79,Cancelled,M,7/8/2024 2:14:00,,,,1 +103,80,Complete,F,3/20/2023 1:18:00,,3/22/2023 2:33:00,3/22/2023 5:01:00,1 +104,82,Complete,F,1/12/2024 6:08:00,,1/13/2024 13:52:00,1/14/2024 14:23:00,1 +105,83,Cancelled,F,9/28/2023 10:22:00,,,,2 +106,84,Shipped,F,9/30/2023 12:59:00,,10/1/2023 1:17:00,,1 +107,84,Complete,F,6/30/2021 12:59:00,,6/30/2021 22:40:00,7/3/2021 6:52:00,1 +108,85,Shipped,M,2/24/2024 18:19:00,,2/27/2024 2:55:00,,1 +109,87,Cancelled,F,9/21/2023 3:55:00,,,,3 +110,87,Complete,F,3/15/2022 3:55:00,,3/16/2022 12:46:00,3/18/2022 1:04:00,2 +111,87,Complete,F,2/3/2024 3:55:00,,2/3/2024 16:18:00,2/5/2024 17:35:00,1 +112,88,Shipped,F,10/7/2023 16:33:00,,10/7/2023 23:58:00,,1 +113,89,Sh,F,2/28/2022 9:45:00,,3/1/2022 14:47:00,,1 +114,89,Cancelled,F,1/22/2023 9:45:00,,,,1 +115,91,Processing,F,1/17/2021 10:05:00,,,,1 +116,92,Complete,F,10/29/2020 10:15:00,,10/30/2020 1:11:00,10/30/2020 12:29:00,1 +117,93,Shipped,M,11/13/2023 11:12:00,,11/15/2023 14:30:00,,1 +118,93,Cancelled,M,8/2/2024 11:12:00,,,,1 +119,93,Complete,M,7/19/2024 11:12:00,,7/21/2024 23:06:00,7/26/2024 14:08:00,3 +120,93,Processing,M,5/20/2021 11:12:00,,,,2 +121,94,Complete,F,12/2/2021 17:16:00,,12/5/2021 9:05:00,12/8/2021 18:33:00,1 +122,96,Complete,F,10/10/2022 18:10:00,,10/13/2022 10:59:00,10/17/2022 22:37:00,1 +123,98,Complete,F,6/8/2024 14:59:00,,6/9/2024 21:17:00,6/14/2024 12:06:00,1 +124,98,Returned,F,8/4/2023 14:59:00,8/9/2023 22:57:00,8/5/2023 23:20:00,8/9/2023 13:06:00,1 +125,98,Processing,F,11/7/2023 14:59:00,,,,1 +126,98,Shipped,F,10/16/2021 14:59:00,,10/19/2021 0:36:00,,4 +127,99,Returned,M,6/28/2024 12:28:00,7/3/2024 15:57:00,6/30/2024 8:56:00,6/30/2024 21:05:00,1 +128,106,Complete,F,1/18/2022 17:45:00,,1/19/2022 23:32:00,1/24/2022 16:31:00,1 +129,107,Shipped,F,1/28/2024 9:44:00,,1/29/2024 4:01:00,,1 +130,108,Shipped,F,8/2/2022 0:33:00,,8/3/2022 2:10:00,,1 +131,109,Complete,M,9/1/2022 4:50:00,,9/2/2022 22:56:00,9/7/2022 6:55:00,2 +132,109,Processing,M,11/8/2022 4:50:00,,,,1 +133,110,Complete,F,12/27/2022 16:20:00,,12/27/2022 23:06:00,12/28/2022 9:08:00,2 +134,112,Complete,M,4/15/2022 1:34:00,,4/16/2022 13:33:00,4/19/2022 4:56:00,1 +135,112,Cancelled,M,4/25/2022 1:34:00,,,,1 +136,112,Cancelled,M,8/15/2021 1:34:00,,,,1 +137,112,Complete,M,8/25/2019 1:34:00,,8/27/2019 9:49:00,8/30/2019 12:02:00,2 +138,113,Complete,M,5/13/2022 15:42:00,,5/13/2022 20:10:00,5/15/2022 2:55:00,2 +139,113,Shipped,M,4/18/2021 15:42:00,,4/19/2021 15:53:00,,2 +140,114,Returned,M,12/9/2023 8:03:00,12/15/2023 17:15:00,12/9/2023 9:41:00,12/12/2023 18:45:00,3 +141,115,Complete,M,9/1/2024 0:18:00,,9/3/2024 3:28:00,9/4/2024 2:29:00,1 +142,116,Returned,M,9/8/2022 18:15:00,9/13/2022 14:42:00,9/9/2022 23:20:00,9/11/2022 1:35:00,1 +143,117,Complete,F,7/26/2023 10:20:00,,7/28/2023 14:14:00,7/31/2023 12:00:00,1 +144,117,Shipped,F,4/7/2023 10:20:00,,4/10/2023 0:56:00,,2 +145,118,Complete,M,5/4/2024 5:28:00,,5/6/2024 23:06:00,5/9/2024 17:31:00,2 +146,119,Processing,F,3/4/2023 17:58:00,,,,2 +147,119,Returned,F,12/2/2019 17:58:00,12/6/2019 6:03:00,12/3/2019 7:01:00,12/4/2019 2:23:00,2 +148,121,Processing,F,12/15/2022 17:55:00,,,,1 +149,121,Processing,F,3/29/2021 17:55:00,,,,1 +150,122,Complete,F,3/11/2023 16:07:00,,3/12/2023 2:54:00,3/15/2023 22:49:00,4 +151,122,Processing,F,4/6/2024 16:07:00,,,,1 +152,123,Canc,M,4/12/2023 14:02:00,,,,1 +153,124,Shipped,F,9/11/2023 15:24:00,,9/12/2023 0:17:00,,2 +154,124,Complete,F,4/23/2024 15:24:00,,4/24/2024 4:45:00,4/26/2024 8:56:00,1 +155,124,Complete,F,6/9/2024 15:24:00,,6/11/2024 13:43:00,6/13/2024 10:06:00,1 +156,126,Complete,F,10/27/2022 0:56:00,,10/27/2022 9:28:00,10/29/2022 17:46:00,1 +157,126,Processing,F,8/1/2022 0:56:00,,,,1 +158,127,Shipped,F,4/16/2021 1:09:00,,4/16/2021 6:49:00,,2 +159,128,Processing,M,12/6/2020 7:25:00,,,,1 +160,129,Processing,M,7/13/2024 9:24:00,,,,1 +161,129,Shipped,M,6/29/2024 9:24:00,,7/1/2024 10:43:00,,2 +162,130,Cancelled,M,1/29/2021 13:48:00,,,,1 +163,130,Shipped,M,7/15/2024 13:48:00,,7/15/2024 20:24:00,,1 +164,131,Processing,M,8/4/2021 13:46:00,,,,1 +165,131,Processing,M,5/20/2021 13:46:00,,,,1 +166,131,Cancelled,M,6/14/2022 13:46:00,,,,1 +167,131,Shipped,M,3/9/2023 13:46:00,,3/11/2023 17:58:00,,4 +168,132,Shipped,M,5/30/2024 12:26:00,,5/30/2024 20:27:00,,1 +169,133,Complete,M,5/9/2023 14:52:00,,5/11/2023 0:31:00,5/12/2023 12:57:00,1 +170,134,Processing,F,4/18/2024 14:58:00,,,,1 +171,134,Cancelled,F,4/18/2023 14:58:00,,,,1 +172,136,Processing,M,6/15/2023 18:41:00,,,,2 +173,137,Complete,F,9/14/2020 12:12:00,,9/16/2020 4:57:00,9/19/2020 6:18:00,1 +174,138,Complete,M,8/8/2022 17:28:00,,8/11/2022 2:48:00,8/15/2022 16:58:00,1 +175,140,Shipped,M,6/14/2024 16:50:00,,6/15/2024 9:16:00,,2 +176,141,Complete,F,2/29/2024 11:22:00,,3/1/2024 16:48:00,3/4/2024 3:15:00,2 +177,142,Processing,M,2/13/2023 12:53:00,,,,1 +178,143,Shipped,M,8/28/2023 17:27:00,,8/29/2023 0:57:00,,1 +179,144,Shipped,F,7/29/2024 13:41:00,,8/1/2024 3:10:00,,1 +180,144,Processing,F,7/16/2024 13:41:00,,,,1 +181,145,Processing,M,8/22/2023 6:14:00,,,,1 +182,146,Shipped,M,2/13/2021 0:33:00,,2/14/2021 22:28:00,,1 +183,147,-,M,6/23/2024 5:04:00,,6/24/2024 0:01:00,6/26/2024 18:24:00,1 +184,148,Complete,F,9/30/2023 18:03:00,,9/30/2023 23:37:00,10/3/2023 5:00:00,1 +185,150,Processing,F,6/29/2024 3:23:00,,,,1 +186,150,Cancelled,F,8/1/2024 3:23:00,,,,1 +187,151,Complete,M,3/21/2023 8:08:00,,3/24/2023 1:40:00,3/28/2023 17:03:00,2 +188,152,Shipped,F,6/8/2024 15:59:00,,6/10/2024 6:37:00,,1 +189,153,Shipped,M,7/23/2024 18:08:00,,7/25/2024 19:54:00,,1 +190,154,Returned,M,9/29/2021 1:45:00,10/4/2021 13:55:00,10/1/2021 18:25:00,10/4/2021 13:10:00,4 +191,154,Complete,M,3/5/2024 1:45:00,,3/6/2024 12:58:00,3/8/2024 3:59:00,1 +192,154,Complete,M,9/7/2023 1:45:00,,9/8/2023 21:58:00,9/10/2023 16:15:00,1 +193,155,Shipped,M,8/25/2022 2:23:00,,8/25/2022 10:04:00,,1 +194,156,Shipped,F,12/21/2021 15:30:00,,12/24/2021 15:21:00,,1 +195,157,Shipped,M,4/5/2024 6:55:00,,4/5/2024 21:00:00,,1 +196,157,Processing,M,4/22/2021 6:55:00,,,,2 +197,158,Processing,F,5/14/2021 9:54:00,,,,1 +198,159,Cancelled,M,1/20/2020 0:00:00,,,,1 +199,160,Returned,M,3/30/2024 17:57:00,4/2/2024 20:10:00,4/1/2024 6:26:00,4/1/2024 21:29:00,2 +200,161,Processing,M,6/30/2022 8:34:00,,,,1 +201,163,Returned,M,8/13/2023 7:26:00,8/20/2023 12:06:00,8/14/2023 16:34:00,8/17/2023 22:18:00,2 +202,164,Processing,F,12/12/2023 16:11:00,,,,1 +203,165,Processing,F,6/6/2023 17:47:00,,,,1 +204,165,Shipped,F,8/10/2024 17:47:00,,8/12/2024 23:58:00,,1 +205,166,Complete,F,5/4/2024 10:26:00,,5/4/2024 19:42:00,5/7/2024 10:01:00,1 +206,166,Complete,F,1/26/2023 10:26:00,,1/28/2023 2:06:00,1/31/2023 2:00:00,1 +207,167,Shipped,F,3/13/2024 2:12:00,,3/14/2024 19:48:00,,1 +208,168,Shipped,F,6/16/2023 1:47:00,,6/16/2023 8:56:00,,1 +209,169,Complete,F,12/13/2023 17:38:00,,12/15/2023 12:35:00,12/19/2023 8:42:00,1 +210,169,Complete,F,12/6/2023 17:38:00,,12/8/2023 9:00:00,12/9/2023 17:27:00,1 +211,170,Complete,F,7/31/2024 8:53:00,,8/1/2024 8:43:00,8/6/2024 7:34:00,3 +212,170,Returned,F,4/20/2024 8:53:00,4/23/2024 18:57:00,4/21/2024 13:59:00,4/21/2024 23:49:00,1 +213,170,Cancelled,F,7/7/2024 8:53:00,,,,1 +214,171,Complete,Male,1/23/2024 1:59:00,,1/24/2024 0:07:00,1/25/2024 5:44:00,1 +215,172,Shipped,M,9/4/2023 4:51:00,,9/4/2023 13:16:00,,1 +216,172,Cancelled,M,10/25/2020 4:51:00,,,,1 +217,172,Returned,M,2/22/2022 4:51:00,2/24/2022 17:47:00,2/22/2022 8:26:00,2/22/2022 20:09:00,2 +218,172,,M,11/28/2021 4:51:00,,11/30/2021 13:44:00,12/2/2021 16:46:00,4 +219,173,Cancelled,F,8/2/2021 7:26:00,,,,1 +220,173,Processing,F,12/3/2023 7:26:00,,,,1 +221,175,Shipped,M,8/29/2021 4:42:00,,8/29/2021 22:21:00,,1 +222,176,Processing,M,5/23/2024 4:00:00,,,,1 +223,177,Processing,M,3/13/2024 7:49:00,,,,1 +224,177,Complete,M,12/2/2023 7:49:00,,12/5/2023 2:06:00,12/6/2023 22:40:00,3 +225,179,Complete,M,2/10/2024 17:17:00,,2/11/2024 5:39:00,2/15/2024 20:53:00,1 +226,180,Returned,F,3/12/2023 7:20:00,3/16/2023 10:37:00,3/13/2023 21:35:00,3/15/2023 5:06:00,1 +227,181,Complete,F,1/9/2023 18:51:00,,1/12/2023 3:54:00,1/14/2023 8:40:00,1 +228,182,Complete,M,1/19/2023 3:29:00,,1/19/2023 18:33:00,1/21/2023 15:43:00,2 +229,183,Processing,M,1/30/2023 4:52:00,,,,1 +230,183,Complete,M,5/8/2024 4:52:00,,5/10/2024 3:27:00,5/10/2024 9:12:00,3 +231,186,Complete,F,9/7/2024 2:02:40,,9/9/2024 1:23:40,9/13/2024 23:57:40,1 +232,187,Processing,F,11/14/2021 16:02:00,,,,1 +233,188,Complete,F,4/24/2024 8:24:00,,4/24/2024 23:24:00,4/28/2024 22:03:00,2 +234,189,Cancelled,F,3/24/2024 15:52:00,,,,1 +235,189,Cancelled,F,3/1/2022 15:52:00,,,,1 +236,190,Shipped,F,9/4/2024 11:50:00,,9/4/2024 23:28:00,,2 +237,191,Complete,F,4/16/2024 8:45:00,,4/18/2024 11:50:00,4/23/2024 0:37:00,1 +238,192,Complete,F,3/16/2022 13:03:00,,3/19/2022 11:32:00,3/21/2022 12:54:00,1 +239,193,Processing,M,3/30/2024 5:33:00,,,,1 +240,193,Returned,M,8/9/2024 5:33:00,8/16/2024 17:22:00,8/11/2024 19:20:00,8/16/2024 16:14:00,1 +241,194,Shipped,M,11/8/2020 10:35:00,,11/9/2020 3:02:00,,2 +242,196,Returned,M,8/15/2024 10:19:00,8/18/2024 18:07:00,8/18/2024 7:13:00,8/18/2024 7:26:00,2 +243,197,Cancelled,M,8/30/2024 8:10:00,,,,1 +244,197,Cancelled,M,8/22/2024 8:10:00,,,,2 +245,198,Complete,M,7/9/2023 3:45:00,,7/11/2023 12:23:00,7/13/2023 19:36:00,4 +246,200,Processing,M,8/10/2021 4:59:00,,,,2 +247,202,Returned,F,4/10/2024 1:57:00,4/14/2024 1:24:00,4/12/2024 1:53:00,4/12/2024 11:51:00,1 +248,202,Complete,F,1/9/2024 1:57:00,,1/9/2024 21:47:00,1/11/2024 11:26:00,1 +249,203,Cancelled,M,7/15/2023 1:03:00,,,,4 +250,204,Processing,F,3/6/2023 10:50:00,,,,3 +251,205,Complete,M,4/20/2024 7:21:00,,4/22/2024 7:51:00,4/25/2024 22:14:00,1 +252,205,Shipped,M,8/16/2023 7:21:00,,8/18/2023 6:14:00,,1 +253,206,Processing,M,1/11/2024 12:35:00,,,,1 +254,207,Complete,F,1/27/2023 0:13:00,,1/28/2023 14:00:00,1/30/2023 10:35:00,4 +255,208,Complete,F,9/8/2024 18:14:00,,9/10/2024 8:26:00,9/15/2024 1:11:00,1 +256,210,Processing,M,2/21/2023 13:19:00,,,,1 +257,210,Complete,M,8/19/2021 13:19:00,,8/20/2021 6:23:00,8/24/2021 4:26:00,1 +258,211,Cancelled,F,8/1/2024 12:13:00,,,,4 +259,212,Shipped,M,7/29/2024 1:12:00,,7/29/2024 10:27:00,,2 +260,212,Cancelled,M,12/18/2023 1:12:00,,,,1 +261,213,Shipped,F,11/7/2023 14:28:00,,11/9/2023 9:55:00,,1 +262,213,Complete,F,11/19/2022 14:28:00,,11/21/2022 4:26:00,11/24/2022 20:08:00,4 +263,213,Complete,F,5/9/2024 14:28:00,,5/11/2024 14:11:00,5/16/2024 13:06:00,1 +264,214,Shipped,M,7/24/2022 11:16:00,,7/26/2022 18:30:00,,1 +265,215,Complete,F,3/22/2024 5:52:00,,3/22/2024 7:48:00,3/24/2024 13:08:00,1 +266,216,Returned,F,1/21/2024 5:27:00,1/26/2024 20:51:00,1/23/2024 21:00:00,1/24/2024 20:08:00,2 +267,216,Cancelled,F,6/20/2024 5:27:00,,,,1 +268,217,Cancelled,M,5/13/2020 10:31:00,,,,1 +269,220,Returned,M,6/23/2021 12:37:00,6/27/2021 9:51:00,6/24/2021 3:15:00,6/26/2021 20:00:00,2 +270,220,Shipped,M,8/7/2024 12:37:00,,8/8/2024 8:29:00,,1 +271,222,Cancelled,M,2/3/2024 10:55:00,,,,3 +272,223,Shipped,F,4/11/2023 15:07:00,,4/12/2023 22:18:00,,2 +273,223,Shipped,F,3/12/2023 15:07:00,,3/14/2023 8:56:00,,1 +274,225,Processing,M,1/14/2024 12:09:00,,,,1 +275,225,Shipped,M,7/30/2024 12:09:00,,8/2/2024 10:40:00,,2 +276,226,Cancelled,M,7/16/2024 15:31:00,,,,4 +277,229,Complete,M,12/27/2023 3:01:00,,12/28/2023 14:28:00,1/2/2024 8:26:00,1 +278,229,Processing,M,2/6/2024 3:01:00,,,,1 +279,230,Returned,M,5/18/2024 2:35:00,5/25/2024 11:45:00,5/20/2024 18:02:00,5/23/2024 12:27:00,4 +280,230,Returned,M,8/10/2024 2:35:00,8/16/2024 13:24:00,8/12/2024 16:45:00,8/16/2024 5:54:00,1 +281,230,Shipped,M,6/9/2024 2:35:00,,6/10/2024 11:24:00,,4 +282,231,Complete,M,6/16/2024 8:12:00,,6/16/2024 18:33:00,6/17/2024 0:20:00,4 +283,231,Shipped,M,8/18/2023 8:12:00,,8/18/2023 16:17:00,,1 +284,234,Shipped,M,8/22/2023 1:41:00,,8/23/2023 18:17:00,,2 +285,234,Returned,M,7/9/2023 1:41:00,7/12/2023 0:28:00,7/10/2023 7:22:00,7/10/2023 9:09:00,1 +286,235,Cancelled,F,11/11/2022 9:50:00,,,,1 +287,236,Complete,M,5/8/2023 17:41:00,,5/11/2023 1:52:00,5/13/2023 16:17:00,3 +288,237,Returned,F,6/9/2024 17:23:00,6/17/2024 3:55:00,6/11/2024 18:38:00,6/14/2024 9:45:00,1 +289,237,Processing,F,1/12/2023 17:23:00,,,,2 +290,238,Complete,F,8/30/2024 12:14:00,,8/30/2024 20:25:00,9/2/2024 3:20:00,1 +291,238,Shipped,F,8/12/2024 12:14:00,,8/12/2024 16:01:00,,1 +292,239,Cancelled,F,6/28/2023 18:39:00,,,,1 +293,240,Processing,M,11/1/2023 3:55:00,,,,1 +294,240,Complete,M,2/5/2022 3:55:00,,2/6/2022 6:13:00,2/8/2022 5:43:00,1 +295,241,Processing,M,11/15/2022 3:56:00,,,,1 +296,242,Returned,M,9/10/2023 11:54:00,9/15/2023 3:23:00,9/11/2023 2:15:00,9/14/2023 12:47:00,1 +297,242,Shipped,M,3/26/2021 11:54:00,,3/28/2021 12:26:00,,4 +298,242,Returned,M,10/31/2023 11:54:00,11/9/2023 5:50:00,11/2/2023 8:39:00,11/7/2023 1:24:00,1 +299,243,Processing,F,7/29/2023 6:02:00,,,,1 +300,244,Processing,F,3/1/2023 12:56:00,,,,4 +301,244,Complete,F,7/28/2023 12:56:00,,7/30/2023 3:50:00,7/31/2023 17:11:00,1 +302,245,Shipped,M,9/6/2024 3:57:00,,9/8/2024 2:19:00,,1 +303,246,Complete,F,7/4/2023 14:26:00,,7/5/2023 4:36:00,7/8/2023 19:54:00,1 +304,246,Cancelled,F,8/7/2024 14:26:00,,,,1 +305,247,Complete,M,1/31/2022 1:06:00,,2/1/2022 15:22:00,2/2/2022 11:28:00,1 +306,247,Complete,M,9/11/2022 1:06:00,,9/11/2022 11:25:00,9/14/2022 18:21:00,1 +307,247,Processing,M,9/18/2022 1:06:00,,,,1 +308,247,Returned,M,5/14/2023 1:06:00,5/20/2023 16:47:00,5/17/2023 0:55:00,5/19/2023 7:29:00,2 +309,248,Processing,F,1/28/2024 6:57:00,,,,1 +310,248,Cancelled,F,2/5/2024 6:57:00,,,,1 +311,249,Cancelled,F,11/23/2023 15:11:00,,,,2 +312,249,Shipped,F,2/3/2024 15:11:00,,2/5/2024 18:34:00,,1 +313,251,Complete,F,5/4/2024 9:35:00,,5/5/2024 15:20:00,5/6/2024 5:22:00,1 +314,251,Complete,F,7/2/2024 9:35:00,,7/4/2024 23:03:00,7/8/2024 22:33:00,4 +315,251,Cancelled,F,10/15/2022 9:35:00,,,,2 +316,252,Returned,F,6/14/2024 0:26:00,6/16/2024 16:36:00,6/15/2024 11:50:00,6/15/2024 16:53:00,1 +317,253,Complete,F,5/17/2024 7:22:00,,5/18/2024 20:19:00,5/23/2024 10:20:00,3 +318,254,Processing,M,4/24/2020 15:34:00,,,,1 +319,255,Cancelled,F,11/29/2023 18:29:00,,,,1 +320,256,Complete,F,8/11/2023 3:41:00,,8/13/2023 20:36:00,8/16/2023 7:40:00,1 +321,257,Complete,M,7/6/2024 18:43:00,,7/8/2024 20:01:00,7/13/2024 6:41:00,2 +322,257,Processing,M,4/18/2024 18:43:00,,,,2 +323,258,Processing,F,8/21/2024 12:20:00,,,,1 +324,259,Processing,F,7/22/2024 6:09:00,,,,2 +325,260,Shipped,F,9/8/2024 18:40:00,,9/10/2024 22:30:00,,2 +326,260,Shipped,F,9/9/2024 18:40:00,,9/10/2024 0:33:00,,3 +327,261,Returned,M,6/7/2024 12:53:00,6/10/2024 15:15:00,6/7/2024 16:05:00,6/10/2024 9:55:00,1 +328,262,Complete,F,4/2/2024 1:27:00,,4/2/2024 5:37:00,4/6/2024 15:56:00,1 +329,262,Returned,F,8/4/2021 1:27:00,8/9/2021 22:04:00,8/6/2021 20:24:00,8/7/2021 4:41:00,3 +330,262,Complete,F,1/25/2022 1:27:00,,1/27/2022 18:06:00,1/29/2022 20:06:00,1 +331,262,Returned,F,9/28/2020 1:27:00,10/5/2020 22:28:00,9/29/2020 18:19:00,10/4/2020 17:24:00,1 +332,263,Cancelled,F,9/17/2023 7:29:00,,,,3 +333,265,Shipped,F,3/31/2020 11:32:00,,3/31/2020 21:36:00,,1 +334,265,Shipped,F,6/29/2024 11:32:00,,7/1/2024 8:51:00,,2 +335,266,Complete,F,2/28/2023 13:55:00,,3/2/2023 0:43:00,3/2/2023 20:42:00,1 +336,267,Shipped,M,4/16/2020 4:47:00,,4/18/2020 16:33:00,,1 +337,268,Shipped,M,1/5/2020 10:04:00,,1/8/2020 4:28:00,,4 +338,269,Processing,F,7/19/2021 14:13:00,,,,2 +339,269,Shipped,F,12/2/2022 14:13:00,,12/4/2022 4:21:00,,1 +340,270,Returned,F,8/23/2023 15:16:00,8/30/2023 18:34:00,8/24/2023 16:12:00,8/29/2023 10:41:00,1 +341,271,Cancelled,M,1/5/2024 16:19:00,,,,1 +342,271,Complete,M,3/30/2024 16:19:00,,4/2/2024 13:20:00,4/4/2024 1:43:00,3 +343,274,Complete,F,5/22/2021 2:21:00,,5/22/2021 6:07:00,5/26/2021 0:09:00,1 +344,275,Shipped,M,5/10/2024 2:44:00,,5/10/2024 12:16:00,,1 +345,278,Cancelled,M,4/4/2022 4:46:00,,,,1 +346,279,Processing,F,5/1/2024 5:07:00,,,,1 +347,281,Shipped,M,3/15/2022 5:13:00,,3/17/2022 10:15:00,,1 +348,282,Processing,M,5/14/2022 17:17:00,,,,2 +349,283,Shipped,M,6/6/2024 11:10:00,,6/8/2024 12:51:00,,1 +350,284,Shipped,F,3/23/2022 7:02:00,,3/25/2022 2:28:00,,1 +351,284,Complete,F,9/21/2022 7:02:00,,9/22/2022 23:08:00,9/27/2022 5:21:00,1 +352,285,Cancelled,F,7/30/2023 11:54:00,,,,1 +353,285,Shipped,F,9/26/2023 11:54:00,,9/28/2023 17:22:00,,1 +354,287,Processing,F,12/26/2021 6:08:00,,,,2 +355,287,Cancelled,F,6/26/2021 6:08:00,,,,1 +356,289,Complete,M,8/9/2023 9:03:00,,8/10/2023 7:59:00,8/10/2023 11:42:00,3 +357,292,Cancelled,M,10/13/2022 9:56:00,,,,4 +358,293,Returned,F,9/8/2024 8:44:00,9/12/2024 15:01:00,9/8/2024 9:54:00,9/11/2024 14:39:00,1 +359,294,Shipped,M,9/3/2024 13:16:00,,9/5/2024 3:58:00,,1 +360,295,Returned,M,9/18/2021 14:15:00,9/25/2021 7:34:00,9/19/2021 6:41:00,9/22/2021 12:34:00,1 +361,296,Shipped,F,7/26/2024 1:36:00,,7/28/2024 14:06:00,,1 +362,297,Complete,M,6/1/2024 4:37:00,,6/2/2024 4:22:00,6/4/2024 8:41:00,1 +363,298,Complete,M,9/8/2022 6:16:00,,9/11/2022 6:00:00,9/13/2022 22:30:00,2 +364,299,Processing,F,5/3/2023 0:07:00,,,,1 +365,301,Complete,M,7/19/2024 11:02:00,,7/22/2024 6:19:00,7/26/2024 8:42:00,1 +366,302,Cancelled,F,5/30/2024 14:22:00,,,,3 +367,302,Complete,F,11/24/2023 14:22:00,,11/27/2023 10:22:00,12/2/2023 1:37:00,3 +368,303,Complete,F,3/24/2024 9:41:00,,3/26/2024 13:35:00,3/29/2024 15:39:00,1 +369,303,Shipped,F,4/21/2024 9:41:00,,4/22/2024 20:49:00,,2 +370,304,Shipped,M,12/30/2023 0:45:00,,1/1/2024 14:55:00,,1 +371,307,Cancelled,F,7/20/2024 15:18:00,,,,1 +372,308,Complete,F,10/10/2023 5:26:00,,10/10/2023 17:57:00,10/12/2023 0:13:00,2 +373,309,Processing,M,1/26/2024 3:13:00,,,,2 +374,309,Shipped,M,9/28/2022 3:13:00,,9/28/2022 15:25:00,,1 +375,310,Cancelled,M,3/31/2022 6:58:00,,,,1 +376,311,Processing,M,8/9/2023 0:31:00,,,,1 +377,311,Shipped,M,2/10/2024 0:31:00,,2/12/2024 3:27:00,,1 +378,311,Shipped,M,1/26/2024 0:31:00,,1/28/2024 9:00:00,,2 +379,311,Cancelled,M,7/25/2024 0:31:00,,,,1 +380,312,Complete,M,5/12/2023 4:12:00,,5/12/2023 11:09:00,5/15/2023 0:25:00,2 +381,313,Complete,M,8/13/2024 5:47:00,,8/14/2024 15:26:00,8/19/2024 6:53:00,1 +382,313,Cancelled,M,6/9/2024 5:47:00,,,,1 +383,314,Complete,M,8/30/2024 16:42:00,,8/31/2024 18:18:00,9/2/2024 15:46:00,1 +384,315,Processing,F,7/2/2021 11:44:00,,,,1 +385,315,Complete,F,9/30/2020 11:44:00,,10/1/2020 15:38:00,10/3/2020 18:46:00,1 +386,316,Shipped,M,4/6/2024 13:09:00,,4/7/2024 23:03:00,,2 +387,317,Cancelled,M,4/26/2024 2:25:00,,,,1 +388,318,Shipped,F,8/5/2021 8:26:00,,8/5/2021 19:31:00,,1 +389,319,Processing,F,6/6/2022 2:20:00,,,,1 +390,320,Processing,F,11/24/2023 14:24:00,,,,2 +391,322,Cancelled,M,9/10/2024 15:16:43,,,,1 +392,325,Complete,F,3/11/2024 11:34:00,,3/14/2024 2:39:00,3/15/2024 8:59:00,1 +393,326,Complete,F,11/24/2023 18:04:00,,11/25/2023 8:32:00,11/27/2023 15:47:00,2 +394,327,Shipped,M,11/4/2021 7:28:00,,11/6/2021 7:45:00,,4 +395,327,Cancelled,M,7/29/2021 7:28:00,,,,1 +396,328,Processing,M,3/12/2022 4:59:00,,,,1 +397,329,Complete,M,5/25/2023 10:55:00,,5/26/2023 18:31:00,5/31/2023 16:50:00,1 +398,329,Returned,M,12/9/2020 10:55:00,12/14/2020 15:10:00,12/11/2020 10:09:00,12/12/2020 11:51:00,3 +399,330,Cancelled,F,1/1/2022 0:28:00,,,,2 +400,330,Complete,F,12/20/2023 0:28:00,,12/20/2023 4:04:00,12/20/2023 13:53:00,1 +401,330,Complete,F,11/14/2021 0:28:00,,11/16/2021 17:48:00,11/19/2021 10:39:00,1 +402,331,Shipped,F,3/22/2022 17:16:00,,3/24/2022 15:42:00,,2 +403,331,Processing,F,8/7/2021 17:16:00,,,,1 +404,332,Processing,F,12/27/2023 18:28:00,,,,3 +405,332,Cancelled,F,12/29/2023 18:28:00,,,,3 +406,332,Complete,F,12/9/2023 18:28:00,,12/12/2023 3:48:00,12/16/2023 4:47:00,1 +407,333,Shipped,M,7/30/2021 9:58:00,,7/31/2021 19:57:00,,1 +408,333,Shipped,M,11/21/2022 9:58:00,,11/23/2022 5:49:00,,1 +409,334,Shipped,F,9/11/2024 9:51:44,,9/11/2024 13:19:44,,1 +410,335,Shipped,M,1/24/2024 9:02:00,,1/25/2024 1:05:00,,1 +411,336,Complete,F,2/11/2023 1:33:00,,2/13/2023 19:22:00,2/14/2023 20:49:00,1 +412,336,Cancelled,F,9/12/2023 1:33:00,,,,1 +413,337,Shipped,M,4/22/2024 13:09:00,,4/25/2024 10:07:00,,1 +414,337,Processing,M,6/3/2024 13:09:00,,,,4 +415,338,Complete,F,3/5/2023 10:35:00,,3/8/2023 0:50:00,3/8/2023 17:20:00,2 +416,339,Cancelled,M,3/10/2022 5:02:00,,,,1 +417,340,Cancelled,F,1/29/2023 2:55:00,,,,1 +418,341,Processing,M,6/23/2023 12:39:00,,,,2 +419,341,Processing,M,4/5/2021 12:39:00,,,,1 +420,342,Shipped,M,10/2/2023 6:35:00,,10/2/2023 9:00:00,,1 +421,342,Returned,M,12/3/2023 6:35:00,12/10/2023 16:00:00,12/5/2023 5:30:00,12/8/2023 23:39:00,1 +422,343,Complete,M,9/8/2024 2:53:44,,9/8/2024 8:36:44,9/11/2024 15:59:44,1 +423,343,Processing,M,9/6/2024 2:53:44,,,,3 +424,344,Shipped,M,8/21/2024 2:55:00,,8/22/2024 21:24:00,,1 +425,345,Complete,M,9/10/2024 5:19:44,,9/10/2024 21:33:44,9/12/2024 15:05:44,2 +426,346,Shipped,M,7/19/2024 13:57:00,,7/20/2024 18:17:00,,1 +427,346,Cancelled,M,9/8/2024 13:57:00,,,,1 +428,347,Returned,F,6/29/2024 18:11:00,7/5/2024 17:25:00,7/1/2024 22:13:00,7/4/2024 10:42:00,1 +429,347,Complete,F,10/15/2022 18:11:00,,10/17/2022 8:39:00,10/21/2022 3:04:00,1 +430,349,Processing,F,1/16/2023 13:08:00,,,,3 +431,349,Cancelled,F,4/27/2022 13:08:00,,,,1 +432,350,Shipped,M,11/2/2021 3:07:00,,11/3/2021 11:19:00,,1 +433,353,Shipped,F,2/19/2023 12:39:00,,2/20/2023 10:17:00,,1 +434,355,Shipped,M,9/30/2021 8:20:00,,10/2/2021 6:11:00,,1 +435,356,Cancelled,F,4/27/2022 14:40:00,,,,2 +436,356,Complete,F,10/12/2020 14:40:00,,10/13/2020 0:23:00,10/16/2020 10:09:00,1 +437,358,Complete,F,4/29/2024 4:01:00,,5/1/2024 14:47:00,5/4/2024 19:28:00,2 +438,360,Complete,F,5/9/2023 1:36:00,,5/10/2023 9:46:00,5/12/2023 5:00:00,1 +439,361,Shipped,F,8/15/2023 13:00:00,,8/16/2023 10:29:00,,1 +440,362,Complete,M,6/13/2024 17:46:00,,6/16/2024 12:53:00,6/20/2024 9:13:00,2 +441,363,Shipped,M,1/22/2022 15:54:00,,1/23/2022 8:38:00,,1 +442,363,Shipped,M,9/5/2020 15:54:00,,9/7/2020 23:37:00,,1 +443,364,Cancelled,M,1/2/2022 2:54:00,,,,2 +444,365,Processing,F,4/12/2024 2:34:00,,,,1 +445,367,Processing,F,2/4/2020 17:58:00,,,,1 +446,367,Shipped,F,11/26/2023 17:58:00,,11/28/2023 20:01:00,,1 +447,368,Processing,F,7/15/2022 11:40:00,,,,2 +448,369,Returned,F,1/12/2024 11:48:00,1/16/2024 17:52:00,1/12/2024 12:07:00,1/14/2024 20:12:00,1 +449,370,Processing,F,10/3/2021 10:57:00,,,,1 +450,371,Cancelled,M,1/17/2023 16:17:00,,,,1 +451,372,Shipped,M,6/11/2024 9:30:00,,6/13/2024 8:12:00,,4 +452,373,Shipped,F,1/19/2024 9:55:00,,1/22/2024 9:28:00,,2 +453,374,Cancelled,F,12/20/2022 17:01:00,,,,1 +454,374,Complete,F,9/7/2024 17:01:00,,9/10/2024 8:42:00,9/13/2024 2:17:00,1 +455,375,Shipped,F,11/24/2023 4:47:00,,11/24/2023 20:25:00,,1 +456,376,Complete,M,8/9/2024 4:41:00,,8/9/2024 7:57:00,8/9/2024 10:17:00,1 +457,376,Complete,M,8/18/2024 4:41:00,,8/19/2024 12:25:00,8/22/2024 9:35:00,2 +458,378,Cancelled,F,8/1/2024 2:39:00,,,,1 +459,379,Complete,M,12/30/2023 5:18:00,,1/1/2024 5:11:00,1/3/2024 0:12:00,2 +460,379,Complete,M,9/9/2023 5:18:00,,9/12/2023 2:04:00,9/12/2023 5:56:00,1 +461,380,Complete,M,9/8/2024 10:14:00,,9/9/2024 17:16:00,9/11/2024 16:50:00,1 +462,381,Shipped,F,5/9/2022 7:17:00,,5/10/2022 1:49:00,,1 +463,385,Processing,M,8/6/2024 17:03:00,,,,4 +464,385,Complete,M,8/24/2024 17:03:00,,8/26/2024 7:31:00,8/26/2024 9:52:00,1 +465,385,Cancelled,M,8/16/2024 17:03:00,,,,2 +466,385,Processing,M,8/13/2024 17:03:00,,,,2 +467,386,Shipped,F,8/16/2024 17:59:00,,8/17/2024 17:57:00,,1 +468,387,Cancelled,M,12/24/2023 3:57:00,,,,1 +469,389,Cancelled,M,8/14/2023 0:15:00,,,,1 +470,390,Shipped,M,7/5/2019 8:09:00,,7/7/2019 5:38:00,,2 +471,391,Processing,M,8/13/2024 13:44:00,,,,1 +472,392,Complete,M,5/10/2024 11:34:00,,5/13/2024 4:27:00,5/17/2024 23:31:00,1 +473,392,Cancelled,M,2/17/2024 11:34:00,,,,1 +474,393,Shipped,F,7/19/2024 13:46:00,,7/19/2024 23:23:00,,1 +475,395,Shipped,M,6/6/2023 10:58:00,,6/6/2023 13:40:00,,2 +476,395,Complete,M,6/30/2022 10:58:00,,6/30/2022 22:56:00,7/2/2022 9:51:00,3 +477,397,Complete,F,11/2/2022 13:37:00,,11/5/2022 2:20:00,11/5/2022 20:46:00,1 +478,397,Shipped,F,7/9/2023 13:37:00,,7/9/2023 22:20:00,,1 +479,398,Shipped,F,11/4/2023 2:59:00,,11/5/2023 10:05:00,,1 +480,399,Processing,M,7/31/2020 9:09:00,,,,3 +481,399,Processing,M,3/8/2021 9:09:00,,,,1 +482,401,Cancelled,F,8/23/2021 6:45:00,,,,1 +483,401,Shipped,F,5/19/2024 6:45:00,,5/20/2024 20:09:00,,2 +484,403,Complete,M,12/12/2021 15:13:00,,12/14/2021 13:07:00,12/19/2021 8:38:00,1 +485,403,Shipped,M,8/16/2023 15:13:00,,8/18/2023 0:00:00,,1 +486,404,Processing,F,8/28/2023 1:27:00,,,,1 +487,404,Processing,F,1/14/2023 1:27:00,,,,1 +488,407,Shipped,M,7/25/2022 16:40:00,,7/26/2022 5:17:00,,4 +489,408,Processing,F,2/4/2024 7:13:00,,,,1 +490,408,Complete,F,1/29/2024 7:13:00,,1/29/2024 18:53:00,2/1/2024 1:27:00,1 +491,409,Shipped,F,8/18/2024 15:22:00,,8/19/2024 8:18:00,,1 +492,410,Shipped,M,7/1/2024 6:36:00,,7/1/2024 17:51:00,,2 +493,410,Complete,M,8/12/2024 6:36:00,,8/15/2024 0:31:00,8/17/2024 22:09:00,2 +494,410,Shipped,M,10/2/2023 6:36:00,,10/5/2023 5:15:00,,1 +495,412,Cancelled,F,7/8/2024 13:35:00,,,,1 +496,413,Complete,F,4/5/2024 12:59:00,,4/7/2024 23:24:00,4/9/2024 20:49:00,1 +497,413,Processing,F,7/7/2024 12:59:00,,,,4 +498,414,Shipped,M,8/28/2024 18:12:00,,8/29/2024 7:06:00,,1 +499,415,Processing,M,12/11/2020 4:22:00,,,,1 +500,416,Processing,F,12/13/2021 10:11:00,,,,2 +501,417,Shipped,M,8/7/2021 9:43:00,,8/8/2021 5:59:00,,2 +502,418,Complete,M,10/17/2019 4:36:00,,10/17/2019 23:13:00,10/21/2019 22:06:00,2 +503,418,Cancelled,M,6/7/2023 4:36:00,,,,1 +504,419,Shipped,N,8/8/2024 8:35:00,,8/8/2024 9:57:00,,4 +505,420,Shipped,F,3/14/2024 15:18:00,,3/16/2024 3:41:00,,1 +506,420,Cancelled,F,1/20/2021 15:18:00,,,,2 +507,420,Complete,F,9/11/2023 15:18:00,,9/13/2023 9:35:00,9/15/2023 20:14:00,1 +508,421,Complete,F,8/28/2024 11:20:00,,8/29/2024 4:48:00,8/29/2024 12:43:00,1 +509,422,Complete,M,7/30/2024 4:20:00,,8/2/2024 2:08:00,8/5/2024 2:36:00,2 +510,423,Returned,F,2/1/2023 13:33:00,2/8/2023 1:25:00,2/2/2023 18:48:00,2/5/2023 2:29:00,1 +511,424,Shipped,M,1/25/2024 16:09:00,,1/27/2024 22:54:00,,1 +512,424,Complete,M,2/5/2024 16:09:00,,2/6/2024 14:12:00,2/8/2024 15:00:00,1 +513,424,Cancelled,M,6/25/2023 16:09:00,,,,1 +514,425,Shipped,M,8/30/2024 7:02:00,,8/30/2024 7:55:00,,2 +515,426,Shipped,F,6/21/2022 10:37:00,,6/23/2022 0:51:00,,1 +516,426,Cancelled,F,10/21/2021 10:37:00,,,,1 +517,427,Processing,M,2/6/2023 17:08:00,,,,1 +518,427,Returned,M,8/2/2024 17:08:00,8/10/2024 1:14:00,8/4/2024 2:17:00,8/7/2024 9:00:00,1 +519,428,Cancelled,M,10/2/2023 5:45:00,,,,1 +520,429,Shipped,M,7/6/2023 14:52:00,,7/8/2023 15:12:00,,2 +521,431,Processing,Male,7/30/2024 7:07:00,,,,1 +522,432,Complete,?,6/4/2024 5:54:00,,6/4/2024 21:57:00,6/9/2024 18:48:00,1 +523,432,Returned,F,3/11/2024 5:54:00,3/19/2024 20:29:00,3/13/2024 3:52:00,3/17/2024 12:09:00,1 +524,432,Returned,F,3/16/2024 5:54:00,3/20/2024 19:37:00,3/17/2024 20:50:00,3/20/2024 2:54:00,1 +525,434,Shipped,M,3/30/2021 6:04:00,,3/30/2021 14:41:00,,1 +526,435,Complete,M,11/30/2023 14:05:00,,12/1/2023 5:02:00,12/3/2023 19:40:00,1 +527,436,Complete,F,11/20/2023 12:21:00,,11/23/2023 2:18:00,11/23/2023 8:07:00,1 +528,436,Complete,F,9/16/2023 12:21:00,,9/16/2023 14:39:00,9/21/2023 2:06:00,1 +529,437,Complete,F,8/26/2024 8:27:00,,8/27/2024 17:25:00,8/30/2024 8:46:00,1 +530,438,Complete,M,3/15/2021 12:29:00,,3/17/2021 5:25:00,3/18/2021 20:14:00,1 +531,439,Returned,F,8/2/2024 5:06:00,8/5/2024 9:14:00,8/4/2024 0:43:00,8/5/2024 0:20:00,2 +532,440,Processing,F,9/27/2020 8:43:00,,,,1 +533,440,Shipped,F,2/13/2021 8:43:00,,2/14/2021 12:52:00,,3 +534,441,Processing,M,12/3/2022 11:42:00,,,,1 +535,441,Complete,M,6/25/2024 11:42:00,,6/27/2024 12:53:00,7/2/2024 9:27:00,1 +536,441,Returned,M,2/8/2023 11:42:00,2/13/2023 3:27:00,2/11/2023 10:22:00,2/12/2023 18:15:00,1 +537,441,Shipped,M,4/13/2022 11:42:00,,4/15/2022 9:13:00,,2 +538,442,Processing,M,7/22/2024 18:51:00,,,,1 +539,443,Complete,F,6/17/2024 7:06:00,,6/17/2024 11:49:00,6/20/2024 11:20:00,2 +540,443,Cancelled,F,3/25/2023 7:06:00,,,,2 +541,444,Complete,F,9/7/2024 4:20:00,,9/9/2024 5:46:00,9/12/2024 17:04:00,1 +542,444,Shipped,F,7/26/2024 4:20:00,,7/29/2024 2:57:00,,1 +543,444,Processing,F,8/22/2024 4:20:00,,,,2 +544,445,Processing,M,4/23/2023 13:58:00,,,,1 +545,446,Shipped,M,11/28/2022 12:54:00,,11/30/2022 5:47:00,,2 +546,446,Shipped,M,1/29/2024 12:54:00,,2/1/2024 1:18:00,,1 +547,448,Shipped,M,9/9/2024 16:04:46,,9/10/2024 13:57:46,,1 +548,449,Cancelled,M,4/22/2024 10:44:00,,,,1 +549,450,Processing,F,9/27/2022 15:12:00,,,,2 +550,450,Returned,F,1/1/2024 15:12:00,1/8/2024 4:30:00,1/3/2024 12:07:00,1/6/2024 1:25:00,2 +551,451,Shipped,F,5/16/2022 10:08:00,,5/18/2022 10:57:00,,1 +552,451,Returned,F,1/18/2023 10:08:00,1/24/2023 10:15:00,1/20/2023 10:55:00,1/22/2023 22:17:00,1 +553,455,Shipped,M,3/10/2023 6:33:00,,3/11/2023 2:42:00,,1 +554,456,Processing,M,12/16/2020 17:49:00,,,,1 +555,457,Returned,F,5/24/2024 5:24:00,5/31/2024 23:21:00,5/27/2024 1:55:00,5/29/2024 19:15:00,1 +556,459,Shipped,F,12/15/2020 14:04:00,,12/17/2020 15:16:00,,2 +557,460,Processing,M,6/3/2024 4:32:00,,,,2 +558,460,Shipped,M,12/10/2023 4:32:00,,12/10/2023 6:39:00,,1 +559,462,Shipped,M,2/18/2023 0:21:00,,2/20/2023 17:36:00,,1 +560,462,Cancelled,M,3/2/2023 0:21:00,,,,1 +561,464,Processing,M,8/4/2022 18:36:00,,,,3 +562,465,Shipped,F,9/18/2022 5:09:00,,9/18/2022 22:10:00,,1 +563,466,Processing,M,5/26/2023 3:32:00,,,,1 +564,468,Complete,F,1/8/2024 3:27:00,,1/9/2024 5:17:00,1/9/2024 6:00:00,1 +565,468,Complete,F,7/4/2022 3:27:00,,7/6/2022 11:43:00,7/10/2022 0:57:00,2 +566,471,Complete,F,7/27/2024 13:45:00,,7/28/2024 15:53:00,7/30/2024 18:42:00,1 +567,472,Shipped,F,6/3/2021 3:02:00,,6/5/2021 13:53:00,,2 +568,473,Processing,F,9/4/2024 15:04:00,,,,1 +569,474,Returned,M,1/14/2022 15:38:00,1/18/2022 6:42:00,1/15/2022 6:37:00,1/15/2022 13:29:00,1 +570,475,Shipped,M,5/8/2023 4:55:00,,5/10/2023 20:51:00,,3 +571,475,Returned,M,4/23/2024 4:55:00,4/30/2024 20:51:00,4/26/2024 2:22:00,4/30/2024 18:47:00,1 +572,475,Cancelled,M,7/18/2023 4:55:00,,,,1 +573,476,Cancelled,F,6/5/2023 3:58:00,,,,1 +574,477,Shipped,F,6/5/2022 0:55:00,,6/7/2022 8:00:00,,1 +575,477,Shipped,F,5/12/2022 0:55:00,,5/14/2022 22:39:00,,2 +576,477,Shipped,F,1/22/2023 0:55:00,,1/22/2023 6:40:00,,1 +577,477,Cancelled,F,4/17/2022 0:55:00,,,,2 +578,478,Complete,F,4/28/2022 8:46:00,,4/29/2022 16:14:00,4/30/2022 2:24:00,1 +579,479,Cancelled,M,12/15/2023 7:13:00,,,,1 +580,479,Complete,M,10/31/2023 7:13:00,,11/2/2023 1:41:00,11/6/2023 1:16:00,1 +581,479,Cancelled,M,12/26/2021 7:13:00,,,,1 +582,479,Cancelled,M,3/15/2022 7:13:00,,,,4 +583,480,Complete,F,5/2/2023 14:57:00,,5/4/2023 19:30:00,5/8/2023 22:13:00,1 +584,481,Returned,F,12/31/2021 8:07:00,1/5/2022 14:19:00,12/31/2021 22:59:00,1/2/2022 18:32:00,2 +585,481,Processing,F,11/23/2021 8:07:00,,,,1 +586,481,Returned,F,9/4/2022 8:07:00,9/7/2022 22:42:00,9/4/2022 19:05:00,9/7/2022 16:15:00,1 +587,481,Complete,F,8/28/2021 8:07:00,,8/29/2021 21:28:00,8/30/2021 22:20:00,1 +588,482,Complete,M,7/7/2023 0:59:00,,7/9/2023 22:24:00,7/10/2023 20:11:00,1 +589,483,Returned,M,2/5/2021 5:42:00,2/11/2021 13:29:00,2/6/2021 16:52:00,2/9/2021 2:36:00,1 +590,484,Processing,F,8/2/2024 0:40:00,,,,3 +591,484,Complete,F,4/1/2022 0:40:00,,4/1/2022 18:10:00,4/5/2022 13:16:00,2 +592,486,Cancelled,F,11/25/2023 11:10:00,,,,1 +593,486,Cancelled,F,3/17/2020 11:10:00,,,,1 +594,487,Processing,M,7/10/2023 11:24:00,,,,1 +595,488,Processing,M,2/21/2024 14:41:00,,,,2 +596,488,Cancelled,M,2/24/2024 14:41:00,,,,2 +597,489,Returned,M,2/25/2022 8:02:00,2/28/2022 6:23:00,2/26/2022 18:19:00,2/27/2022 23:49:00,1 +598,490,Shipped,M,11/28/2023 0:04:00,,11/30/2023 9:21:00,,1 +599,491,Complete,F,9/26/2023 13:42:00,,9/28/2023 11:36:00,10/1/2023 20:12:00,1 +600,492,Processing,F,4/14/2024 10:45:00,,,,1 +601,492,Shipped,F,11/28/2021 10:45:00,,11/30/2021 15:41:00,,1 +602,492,Shipped,F,3/17/2024 10:45:00,,3/18/2024 5:09:00,,1 +603,492,Complete,F,5/17/2022 10:45:00,,5/18/2022 1:13:00,5/21/2022 23:52:00,1 +604,493,Shipped,M,3/13/2024 17:59:00,,3/16/2024 0:21:00,,2 +605,493,Complete,M,10/18/2023 17:59:00,,10/21/2023 8:25:00,10/22/2023 21:58:00,1 +606,494,Complete,F,9/6/2024 17:26:47,,9/7/2024 1:46:47,9/8/2024 4:33:47,1 +607,494,Processing,F,9/9/2024 17:26:47,,,,1 +608,495,Processing,F,1/30/2023 0:30:00,,,,2 +609,497,Cancelled,M,5/1/2021 3:59:00,,,,1 +610,498,Cancelled,F,9/6/2024 15:16:47,,,,1 +611,498,Complete,F,9/8/2024 15:16:47,,9/10/2024 9:13:47,9/11/2024 5:30:47,1 +612,499,Cancelled,F,9/11/2021 1:43:00,,,,1 +613,499,Processing,F,9/3/2020 1:43:00,,,,1 +614,499,Complete,F,1/3/2022 1:43:00,,1/5/2022 19:29:00,1/10/2022 3:22:00,1 +615,500,Processing,F,10/18/2020 11:27:00,,,,1 +616,501,Complete,M,7/12/2024 2:39:00,,7/12/2024 10:11:00,7/13/2024 19:57:00,2 +617,501,Returned,M,4/6/2024 2:39:00,4/10/2024 22:43:00,4/6/2024 20:50:00,4/10/2024 16:42:00,1 +618,502,Cancelled,F,8/9/2024 16:22:00,,,,1 +619,502,Cancelled,F,4/12/2024 16:22:00,,,,3 +620,502,Shipped,F,6/15/2024 16:22:00,,6/17/2024 17:42:00,,1 +621,504,Shipped,M,12/11/2023 14:48:00,,12/13/2023 2:38:00,,2 +622,505,Complete,F,9/18/2022 17:46:00,,9/20/2022 1:41:00,9/22/2022 16:01:00,2 +623,507,Cancelled,F,8/4/2024 13:03:00,,,,2 +624,507,Shipped,F,2/27/2024 13:03:00,,2/29/2024 18:50:00,,3 +625,509,Cancelled,M,7/19/2024 3:56:00,,,,3 +626,510,Processing,F,12/27/2022 11:58:00,,,,1 +627,511,Complete,F,9/7/2024 2:24:00,,9/9/2024 11:36:00,9/12/2024 22:24:00,2 +628,512,Cancelled,F,7/19/2024 16:38:00,,,,1 +629,513,Returned,F,11/1/2021 6:44:00,11/10/2021 0:09:00,11/3/2021 3:37:00,11/7/2021 23:10:00,1 +630,513,Complete,F,1/7/2020 6:44:00,,1/9/2020 19:05:00,1/10/2020 20:33:00,1 +631,514,Complete,F,1/1/2022 8:13:00,,1/3/2022 11:24:00,1/6/2022 8:03:00,1 +632,514,Shipped,F,7/5/2023 8:13:00,,7/7/2023 4:11:00,,2 +633,515,Complete,M,1/6/2024 12:37:00,,1/8/2024 3:45:00,1/8/2024 7:44:00,1 +634,516,Cancelled,F,1/25/2024 18:29:00,,,,1 +635,516,Shipped,F,6/3/2024 18:29:00,,6/4/2024 17:56:00,,1 +636,518,Complete,M,12/30/2023 5:37:00,,12/30/2023 8:15:00,12/31/2023 8:14:00,1 +637,519,Processing,M,12/16/2023 3:34:00,,,,1 +638,519,Complete,M,12/25/2023 3:34:00,,12/26/2023 2:37:00,12/27/2023 5:23:00,1 +639,519,Cancelled,M,3/16/2023 3:34:00,,,,1 +640,520,Complete,F,2/14/2022 18:37:00,,2/15/2022 22:31:00,2/17/2022 18:04:00,2 +641,521,Processing,M,1/9/2020 11:44:00,,,,1 +642,522,Complete,F,11/2/2023 6:54:00,,11/2/2023 16:08:00,11/7/2023 4:20:00,2 +643,524,Processing,F,12/5/2022 9:10:00,,,,2 +644,525,Shipped,M,10/16/2020 1:51:00,,10/17/2020 11:43:00,,1 +645,526,Processing,F,3/31/2021 6:33:00,,,,1 +646,526,Shipped,F,5/27/2024 6:33:00,,5/27/2024 13:17:00,,3 +647,527,Returned,M,1/17/2024 7:20:00,1/22/2024 9:06:00,1/17/2024 14:41:00,1/21/2024 0:55:00,2 +648,527,Complete,M,2/24/2021 7:20:00,,2/25/2021 12:24:00,3/2/2021 7:24:00,1 +649,527,Shipped,M,10/22/2021 7:20:00,,10/22/2021 12:12:00,,1 +650,528,Shipped,M,1/13/2024 17:46:00,,1/13/2024 22:05:00,,1 +651,529,Processing,F,4/28/2024 8:41:00,,,,2 +652,530,Processing,M,9/26/2022 16:49:00,,,,1 +653,532,Shipped,F,3/6/2023 2:16:00,,3/8/2023 10:18:00,,1 +654,535,Shipped,M,6/27/2022 10:17:00,,6/28/2022 6:33:00,,1 +655,536,Processing,M,11/29/2023 4:00:00,,,,1 +656,536,Complete,M,8/3/2023 4:00:00,,8/4/2023 0:38:00,8/8/2023 4:01:00,1 +657,537,Processing,M,8/28/2020 0:18:00,,,,1 +658,538,Complete,M,7/18/2024 14:38:00,,7/21/2024 3:05:00,7/23/2024 11:42:00,4 +659,539,Cancelled,M,1/2/2022 0:34:00,,,,1 +660,539,Shipped,M,4/1/2023 0:34:00,,4/1/2023 16:09:00,,1 +661,539,Cancelled,M,8/9/2021 0:34:00,,,,1 +662,540,Shipped,M,10/17/2021 13:12:00,,10/20/2021 4:26:00,,3 +663,541,Cancelled,M,11/2/2020 15:20:00,,,,2 +664,542,Returned,M,9/19/2023 2:49:00,9/26/2023 19:03:00,9/21/2023 6:17:00,9/25/2023 21:32:00,1 +665,544,Cancelled,F,8/1/2022 8:24:00,,,,1 +666,545,Complete,M,3/10/2023 0:50:00,,3/12/2023 10:25:00,3/14/2023 20:15:00,2 +667,545,Shipped,M,5/8/2023 0:50:00,,5/10/2023 10:48:00,,1 +668,547,Processing,M,12/20/2021 18:10:00,,,,2 +669,548,Shipped,M,9/6/2021 14:45:00,,9/7/2021 21:07:00,,1 +670,550,Returned,M,11/24/2023 7:41:00,11/30/2023 5:15:00,11/26/2023 16:13:00,11/27/2023 17:04:00,1 +671,550,Complete,M,8/1/2024 7:41:00,,8/3/2024 13:34:00,8/4/2024 17:55:00,1 +672,552,Processing,F,6/20/2021 3:36:00,,,,1 +673,553,Processing,M,2/9/2024 13:13:00,,,,3 +674,554,Processing,M,2/24/2024 10:44:00,,,,1 +675,554,Complete,M,5/15/2024 10:44:00,,5/15/2024 17:07:00,5/18/2024 7:46:00,1 +676,554,Processing,M,4/10/2024 10:44:00,,,,1 +677,555,Shipped,M,4/21/2023 12:18:00,,4/23/2023 9:39:00,,3 +678,556,Processing,F,11/27/2022 13:42:00,,,,2 +679,556,Complete,F,3/18/2022 13:42:00,,3/19/2022 0:45:00,3/19/2022 13:58:00,2 +680,558,Processing,M,9/19/2020 12:43:00,,,,2 +681,558,Cancelled,M,4/11/2022 12:43:00,,,,1 +682,559,Complete,M,4/9/2022 1:44:00,,4/9/2022 8:29:00,4/11/2022 3:01:00,4 +683,559,Processing,M,1/9/2023 1:44:00,,,,1 +684,560,Shipped,M,5/24/2024 1:27:00,,5/26/2024 4:01:00,,2 +685,561,Complete,F,10/27/2022 11:03:00,,10/27/2022 18:44:00,10/31/2022 18:24:00,1 +686,562,Shipped,M,6/22/2022 7:51:00,,6/23/2022 19:05:00,,1 +687,562,Complete,M,3/8/2022 7:51:00,,3/8/2022 9:43:00,3/9/2022 11:38:00,1 +688,565,Complete,M,9/2/2021 0:41:00,,9/3/2021 13:01:00,9/7/2021 17:00:00,1 +689,566,Processing,M,1/6/2020 17:02:00,,,,1 +690,566,Complete,M,2/4/2024 17:02:00,,2/5/2024 14:18:00,2/9/2024 3:49:00,2 +691,567,Complete,M,7/16/2024 7:40:00,,7/17/2024 4:46:00,7/21/2024 20:34:00,1 +692,568,Processing,M,9/9/2024 10:01:48,,,,4 +693,568,Complete,M,9/9/2024 10:01:48,,9/11/2024 18:50:48,9/14/2024 6:57:48,1 +694,568,Shipped,M,9/9/2024 10:01:48,,9/12/2024 1:11:48,,1 +695,568,Shipped,M,9/9/2024 10:01:48,,9/10/2024 2:39:48,,1 +696,572,Shipped,M,9/6/2024 5:01:48,,9/6/2024 9:40:48,,1 +697,572,Shipped,M,9/6/2024 5:01:48,,9/8/2024 9:03:48,,4 +698,572,Complete,M,9/8/2024 5:01:48,,9/10/2024 15:37:48,9/11/2024 23:26:48,1 +699,572,Shipped,M,9/9/2024 5:01:48,,9/12/2024 0:10:48,,3 +700,573,Returned,M,8/6/2020 6:57:00,8/14/2020 6:10:00,8/7/2020 14:07:00,8/12/2020 3:50:00,1 +701,574,Processing,F,7/20/2023 9:53:00,,,,1 +702,574,Complete,F,6/2/2023 9:53:00,,6/4/2023 10:19:00,6/8/2023 13:58:00,1 +703,576,Complete,M,2/3/2024 7:57:00,,2/3/2024 15:58:00,2/4/2024 1:26:00,2 +704,578,Cancelled,M,9/2/2023 3:27:00,,,,1 +705,579,Complete,M,12/10/2023 5:57:00,,12/12/2023 2:57:00,12/14/2023 20:07:00,1 +706,580,Shipped,F,4/7/2022 16:08:00,,4/8/2022 22:44:00,,1 +707,581,Cancelled,F,11/9/2022 14:01:00,,,,1 +708,581,Shipped,F,7/2/2024 14:01:00,,7/3/2024 10:22:00,,1 +709,582,Shipped,F,8/13/2024 9:42:00,,8/15/2024 12:21:00,,4 +710,583,Returned,F,2/10/2024 1:10:00,2/12/2024 2:52:00,2/11/2024 0:13:00,2/11/2024 1:53:00,2 +711,583,Shipped,F,2/5/2024 1:10:00,,2/5/2024 19:42:00,,1 +712,585,Shipped,M,3/8/2024 17:23:00,,3/10/2024 6:26:00,,1 +713,585,Shipped,M,8/29/2024 17:23:00,,8/30/2024 2:13:00,,2 +714,587,Complete,F,8/25/2022 13:12:00,,8/27/2022 10:56:00,8/31/2022 17:48:00,2 +715,587,Processing,F,5/30/2023 13:12:00,,,,1 +716,588,Cancelled,F,12/6/2023 3:29:00,,,,1 +717,591,Complete,M,2/18/2021 8:46:00,,2/20/2021 20:45:00,2/20/2021 20:57:00,1 +718,591,Complete,M,8/7/2022 8:46:00,,8/8/2022 8:03:00,8/12/2022 2:10:00,1 +719,593,Returned,M,1/31/2024 1:47:00,2/7/2024 18:50:00,2/2/2024 6:00:00,2/6/2024 17:31:00,1 +720,596,Complete,F,6/23/2023 1:50:00,,6/24/2023 16:49:00,6/27/2023 0:41:00,1 +721,597,Returned,M,5/3/2024 11:00:00,5/11/2024 1:24:00,5/5/2024 18:44:00,5/10/2024 17:45:00,1 +722,597,Processing,M,12/23/2023 11:00:00,,,,3 +723,598,Returned,F,8/12/2023 10:40:00,8/19/2023 0:21:00,8/14/2023 3:20:00,8/18/2023 20:52:00,1 +724,598,Cancelled,F,6/23/2024 10:40:00,,,,2 +725,598,Shipped,F,11/23/2023 10:40:00,,11/24/2023 23:47:00,,1 +726,598,Complete,F,4/30/2024 10:40:00,,5/1/2024 23:17:00,5/5/2024 11:39:00,1 +727,599,Shipped,M,11/28/2023 8:39:00,,11/28/2023 19:54:00,,1 +728,599,Shipped,M,12/22/2023 8:39:00,,12/23/2023 1:39:00,,1 +729,600,Processing,F,4/17/2023 13:57:00,,,,1 +730,601,Complete,F,9/2/2022 16:47:00,,9/3/2022 23:31:00,9/4/2022 19:09:00,1 +731,601,Complete,F,4/22/2023 16:47:00,,4/24/2023 21:46:00,4/26/2023 16:34:00,1 +732,601,Complete,F,11/29/2022 16:47:00,,11/30/2022 14:25:00,12/1/2022 17:02:00,1 +733,601,Shipped,F,12/26/2022 16:47:00,,12/26/2022 21:27:00,,1 +734,602,Processing,F,8/26/2024 8:47:00,,,,1 +735,603,Shipped,M,6/11/2021 14:27:00,,6/14/2021 4:29:00,,3 +736,604,Processing,M,4/4/2024 2:15:00,,,,2 +737,604,Shipped,M,5/8/2024 2:15:00,,5/10/2024 14:39:00,,2 +738,604,Shipped,M,2/16/2024 2:15:00,,2/16/2024 17:04:00,,2 +739,604,Complete,M,8/8/2024 2:15:00,,8/10/2024 1:01:00,8/10/2024 21:58:00,2 +740,605,Complete,M,1/15/2021 2:04:00,,1/15/2021 14:18:00,1/17/2021 6:40:00,1 +741,606,Shipped,M,9/16/2023 8:04:00,,9/17/2023 0:43:00,,1 +742,607,Complete,F,8/18/2024 2:00:00,,8/18/2024 21:30:00,8/19/2024 4:43:00,1 +743,609,Cancelled,M,2/11/2023 5:08:00,,,,1 +744,609,Complete,M,4/19/2024 5:08:00,,4/22/2024 3:56:00,4/27/2024 2:11:00,1 +745,610,Cancelled,F,5/3/2021 1:26:00,,,,1 +746,610,Complete,F,3/7/2024 1:26:00,,3/7/2024 23:14:00,3/10/2024 14:12:00,4 +747,613,Shipped,M,4/5/2023 17:12:00,,4/6/2023 10:34:00,,2 +748,614,Processing,F,11/3/2020 18:01:00,,,,1 +749,614,Processing,F,10/27/2020 18:01:00,,,,1 +750,614,Complete,F,12/3/2019 18:01:00,,12/6/2019 13:38:00,12/10/2019 2:40:00,1 +751,615,Returned,F,3/3/2020 9:15:00,3/8/2020 6:15:00,3/4/2020 19:44:00,3/6/2020 14:00:00,1 +752,619,Shipped,M,7/7/2024 13:53:00,,7/9/2024 12:52:00,,1 +753,619,Shipped,M,6/8/2024 13:53:00,,6/10/2024 10:38:00,,3 +754,620,Cancelled,,7/19/2023 13:12:00,,,,3 +755,621,Complete,F,3/27/2023 3:15:00,,3/27/2023 20:05:00,4/1/2023 0:27:00,1 +756,622,Shipped,F,11/18/2022 14:10:00,,11/21/2022 4:07:00,,1 +757,624,Shipped,M,5/10/2023 2:49:00,,5/10/2023 10:52:00,,1 +758,625,Shipped,F,11/6/2022 3:00:00,,11/7/2022 2:10:00,,1 +759,625,Shipped,F,6/19/2022 3:00:00,,6/21/2022 17:18:00,,1 +760,626,Complete,F,12/6/2023 7:20:00,,12/6/2023 11:31:00,12/8/2023 6:02:00,1 +761,626,Shipped,F,1/7/2024 7:20:00,,1/8/2024 15:34:00,,2 +762,628,Returned,M,9/7/2024 7:43:49,9/15/2024 7:01:49,9/8/2024 0:35:49,9/12/2024 20:01:49,1 +763,628,Processing,M,9/9/2024 7:43:49,,,,1 +764,630,Complete,M,7/28/2024 16:05:00,,7/31/2024 4:26:00,8/3/2024 10:50:00,1 +765,631,Returned,M,7/27/2024 4:55:00,8/2/2024 1:20:00,7/29/2024 21:04:00,7/30/2024 7:39:00,1 +766,632,Cancelled,M,4/18/2023 16:08:00,,,,1 +767,635,Cancelled,M,7/15/2022 7:03:00,,,,1 +768,635,Complete,M,3/2/2023 7:03:00,,3/5/2023 2:42:00,3/7/2023 18:50:00,1 +769,636,Returned,F,5/17/2024 5:44:00,5/22/2024 10:49:00,5/18/2024 14:00:00,5/19/2024 19:36:00,1 +770,637,Processing,F,6/4/2022 4:02:00,,,,1 +771,637,Shipped,F,7/6/2023 4:02:00,,7/7/2023 0:04:00,,1 +772,638,Shipped,M,3/26/2024 8:49:00,,3/26/2024 15:48:00,,1 +773,639,Processing,M,3/11/2024 0:53:00,,,,1 +774,640,Shipped,F,6/8/2024 12:11:00,,6/9/2024 19:54:00,,2 +775,641,Returned,F,6/25/2024 5:00:00,6/26/2024 14:21:00,6/26/2024 1:14:00,6/26/2024 9:15:00,1 +776,641,Shipped,F,3/10/2022 5:00:00,,3/11/2022 7:42:00,,4 +777,641,Returned,F,2/16/2022 5:00:00,2/22/2022 23:15:00,2/17/2022 23:32:00,2/20/2022 0:34:00,1 +778,643,Processing,M,5/25/2021 8:54:00,,,,1 +779,647,Complete,F,10/16/2021 15:21:00,,10/18/2021 21:16:00,10/20/2021 3:40:00,1 +780,647,Processing,F,2/12/2022 15:21:00,,,,1 +781,648,Complete,M,9/1/2024 8:22:00,,9/3/2024 3:23:00,9/6/2024 17:21:00,1 +782,648,Complete,M,7/29/2024 8:22:00,,7/31/2024 21:45:00,8/2/2024 6:55:00,1 +783,649,Shipped,M,7/23/2022 1:05:00,,7/25/2022 23:56:00,,4 +784,650,Shipped,M,4/9/2024 10:04:00,,4/12/2024 3:09:00,,4 +785,650,Shipped,M,12/30/2020 10:04:00,,1/2/2021 1:22:00,,1 +786,651,Processing,M,11/22/2023 2:29:00,,,,2 +787,652,Processing,F,10/7/2022 13:35:00,,,,1 +788,653,Complete,F,5/2/2024 5:01:00,,5/4/2024 15:29:00,5/9/2024 5:34:00,1 +789,654,Shipped,F,5/16/2024 2:14:00,,5/16/2024 14:57:00,,1 +790,654,Shipped,F,2/5/2023 2:14:00,,2/6/2023 1:06:00,,2 +791,654,Cancelled,F,1/29/2021 2:14:00,,,,2 +792,654,Complete,F,3/4/2020 2:14:00,,3/5/2020 4:02:00,3/7/2020 12:42:00,1 +793,655,Processing,F,10/25/2021 16:32:00,,,,2 +794,655,Complete,F,5/22/2024 16:32:00,,5/23/2024 21:38:00,5/24/2024 8:57:00,1 +795,656,Complete,F,7/23/2024 7:30:00,,7/25/2024 18:06:00,7/27/2024 5:26:00,1 +796,656,Complete,F,12/8/2023 7:30:00,,12/9/2023 23:16:00,12/11/2023 21:15:00,1 +797,657,Shipped,M,5/3/2024 17:28:00,,5/4/2024 15:23:00,,3 +798,658,Shipped,M,12/17/2023 15:29:00,,12/19/2023 15:47:00,,1 +799,659,Returned,F,12/20/2021 18:32:00,12/28/2021 2:12:00,12/23/2021 15:48:00,12/25/2021 23:14:00,1 +800,659,Cancelled,F,1/27/2022 18:32:00,,,,1 +801,661,Processing,M,9/9/2024 17:57:50,,,,1 +802,662,Shipped,F,3/21/2024 4:08:00,,3/24/2024 1:54:00,,1 +803,662,Shipped,F,4/21/2024 4:08:00,,4/21/2024 21:55:00,,1 +804,663,Cancelled,F,6/7/2024 12:25:00,,,,1 +805,664,Cancelled,M,8/11/2023 9:20:00,,,,3 +806,665,Processing,M,1/15/2024 1:33:00,,,,1 +807,666,Shipped,M,1/29/2023 14:23:00,,1/31/2023 0:27:00,,1 +808,666,Complete,M,10/14/2020 14:23:00,,10/15/2020 2:47:00,10/18/2020 11:54:00,2 +809,666,Shipped,M,2/13/2023 14:23:00,,2/14/2023 19:29:00,,1 +808,668,Cancelled,F,10/8/2022 12:24:00,,,,1 +811,669,Returned,F,6/26/2019 15:35:00,7/4/2019 10:06:00,6/28/2019 13:38:00,7/2/2019 12:32:00,1 +812,670,Complete,M,9/15/2023 8:11:00,,9/18/2023 2:20:00,9/19/2023 12:36:00,4 +813,670,Processing,M,9/28/2023 8:11:00,,,,1 +814,671,Shipped,F,1/17/2022 11:54:00,,1/20/2022 7:16:00,,1 +815,671,Cancelled,F,12/1/2021 11:54:00,,,,2 +816,672,Shipped,M,7/21/2024 5:21:00,,7/22/2024 2:32:00,,1 +817,674,Returned,M,7/25/2023 2:35:00,7/29/2023 16:33:00,7/25/2023 14:55:00,7/27/2023 17:20:00,1 +818,674,Shipped,M,10/2/2023 2:35:00,,10/4/2023 11:59:00,,4 +819,675,Proc,F,2/10/2024 1:13:00,,,,1 +820,676,Shipped,M,6/26/2024 17:01:00,,6/27/2024 18:05:00,,2 +821,677,Complete,F,3/7/2023 2:18:00,,3/7/2023 23:13:00,3/10/2023 11:59:00,3 +822,678,Returned,F,8/16/2024 4:45:00,8/24/2024 4:09:00,8/17/2024 11:03:00,8/22/2024 7:45:00,1 +823,678,Shipped,F,1/7/2024 4:45:00,,1/7/2024 7:37:00,,1 +824,680,Complete,F,1/21/2024 5:11:00,,1/23/2024 10:26:00,1/23/2024 13:36:00,1 +825,680,Processing,F,3/4/2023 5:11:00,,,,1 +826,680,Complete,F,4/30/2024 5:11:00,,5/1/2024 15:41:00,5/4/2024 13:16:00,1 +827,680,Processing,F,8/31/2023 5:11:00,,,,1 +828,683,Shipped,F,1/3/2022 18:37:00,,1/6/2022 0:32:00,,1 +829,683,Complete,F,5/14/2022 18:37:00,,5/17/2022 18:22:00,5/20/2022 18:34:00,1 +830,685,Shipped,F,11/19/2023 16:27:00,,11/21/2023 7:10:00,,1 +831,685,Complete,F,10/2/2023 16:27:00,,10/4/2023 14:57:00,10/6/2023 21:25:00,1 +832,685,Shipped,F,12/19/2023 16:27:00,,12/21/2023 11:29:00,,1 +833,686,Returned,M,8/8/2021 5:33:00,8/14/2021 16:39:00,8/10/2021 14:13:00,8/14/2021 1:29:00,1 +834,687,Processing,M,8/7/2022 11:37:00,,,,4 +835,688,Shipped,M,5/3/2022 10:41:00,,5/5/2022 15:32:00,,1 +836,689,Complete,M,5/6/2024 11:45:00,,5/8/2024 3:05:00,5/11/2024 18:38:00,1 +837,689,Shipped,M,6/17/2023 11:45:00,,6/17/2023 20:53:00,,2 +838,689,Cancelled,M,12/31/2021 11:45:00,,,,1 +839,689,Complete,M,1/22/2021 11:45:00,,1/22/2021 22:23:00,1/23/2021 4:54:00,1 +840,690,Complete,F,8/12/2024 10:21:00,,8/14/2024 17:15:00,8/19/2024 4:16:00,3 +841,690,Cancelled,F,4/22/2024 10:21:00,,,,2 +842,690,Processing,F,5/21/2023 10:21:00,,,,3 +843,691,Cancelled,F,7/24/2022 6:08:00,,,,1 +844,693,Shipped,F,5/13/2024 17:04:00,,5/15/2024 11:32:00,,1 +845,694,Processing,F,6/14/2023 2:04:00,,,,2 +846,695,Returned,M,6/22/2021 12:18:00,6/29/2021 2:47:00,6/25/2021 1:43:00,6/28/2021 0:49:00,4 +847,696,Complete,F,8/9/2023 4:10:00,,8/11/2023 20:15:00,8/12/2023 14:58:00,1 +848,696,Returned,F,9/12/2023 4:10:00,9/18/2023 21:42:00,9/12/2023 17:51:00,9/16/2023 21:46:00,1 +849,696,Returned,F,6/22/2023 4:10:00,6/25/2023 5:13:00,6/22/2023 6:06:00,6/23/2023 16:22:00,1 +850,696,Shipped,F,10/20/2022 4:10:00,,10/23/2022 2:09:00,,3 +851,697,Shipped,F,3/24/2023 11:04:00,,3/24/2023 14:04:00,,1 +852,698,Processing,F,9/7/2024 6:32:50,,,,2 +853,698,Shipped,F,9/7/2024 6:32:50,,9/10/2024 1:09:50,,1 +854,700,Cancelled,F,2/4/2023 7:04:00,,,,4 +855,700,Returned,F,4/19/2023 7:04:00,4/23/2023 3:51:00,4/19/2023 14:07:00,4/22/2023 20:56:00,2 +856,701,Complete,F,8/19/2024 18:14:00,,8/20/2024 21:01:00,8/22/2024 17:25:00,2 +857,701,Processing,F,2/4/2024 18:14:00,,,,1 +858,701,Processing,F,8/28/2024 18:14:00,,,,1 +859,703,Processing,M,3/19/2024 11:03:00,,,,2 +860,706,Shipped,M,12/22/2023 14:12:00,,12/24/2023 13:01:00,,1 +861,707,Complete,F,7/7/2024 3:19:00,,7/7/2024 3:58:00,7/10/2024 16:20:00,1 +862,709,Cancelled,F,8/30/2024 15:07:00,,,,1 +863,710,Processing,F,10/6/2023 7:18:00,,,,1 +864,712,Processing,F,9/27/2022 9:33:00,,,,1 +865,713,Processing,F,8/8/2024 14:47:00,,,,1 +866,716,Shipped,M,7/16/2019 5:44:00,,7/17/2019 18:31:00,,1 +867,717,Processing,M,3/29/2023 14:07:00,,,,1 +868,717,Complete,M,6/10/2023 14:07:00,,6/13/2023 8:38:00,6/17/2023 2:34:00,1 +869,718,Complete,M,7/20/2024 13:47:00,,7/23/2024 11:22:00,7/24/2024 11:22:00,3 +870,718,Returned,M,10/17/2022 13:47:00,10/22/2022 12:49:00,10/19/2022 11:47:00,10/21/2022 1:06:00,1 +871,719,Returned,M,5/2/2023 18:37:00,5/10/2023 20:52:00,5/5/2023 5:56:00,5/9/2023 18:51:00,1 +872,720,Cancelled,F,6/25/2024 8:22:00,,,,1 +873,723,Shipped,F,6/21/2024 17:21:00,,6/23/2024 11:49:00,,3 +874,724,Cancelled,F,5/28/2024 6:16:00,,,,1 +875,724,Cancelled,F,6/15/2023 6:16:00,,,,3 +876,725,Cancelled,M,2/10/2024 11:47:00,,,,1 +877,725,Complete,M,7/19/2020 11:47:00,,7/22/2020 7:27:00,7/25/2020 5:45:00,1 +878,726,Shipped,F,8/10/2024 12:49:00,,8/12/2024 19:40:00,,1 +879,726,Complete,F,1/23/2024 12:49:00,,1/25/2024 23:13:00,1/27/2024 14:41:00,1 +880,726,Processing,F,3/9/2024 12:49:00,,,,1 +881,728,Processing,M,9/13/2022 16:21:00,,,,1 +882,728,Shipped,M,8/12/2024 16:21:00,,8/13/2024 9:39:00,,3 +883,729,Shipped,M,12/9/2023 18:07:00,,12/11/2023 16:48:00,,4 +884,730,Complete,F,1/27/2022 0:31:00,,1/28/2022 9:04:00,1/31/2022 20:22:00,1 +885,731,Cancelled,F,11/29/2021 14:51:00,,,,1 +886,731,Shipped,F,10/4/2023 14:51:00,,10/6/2023 13:29:00,,1 +887,732,Complete,M,11/25/2020 6:39:00,,11/26/2020 8:20:00,11/28/2020 0:36:00,1 +888,733,Processing,M,9/9/2024 0:48:51,,,,2 +889,733,Shipped,M,9/8/2024 0:48:51,,9/8/2024 7:21:51,,3 +890,733,Complete,M,9/9/2024 0:48:51,,9/9/2024 21:09:51,9/12/2024 8:14:51,1 +891,734,Shipped,M,10/6/2022 7:05:00,,10/7/2022 9:17:00,,1 +892,735,Complete,M,6/22/2022 18:58:00,,6/24/2022 0:47:00,6/24/2022 23:11:00,1 +893,735,Shipped,M,10/8/2022 18:58:00,,10/10/2022 4:26:00,,1 +894,736,Shipped,F,9/19/2021 1:10:00,,9/20/2021 22:03:00,,1 +895,736,Cancelled,F,2/25/2023 1:10:00,,,,1 +896,738,Complete,M,9/8/2024 12:43:00,,9/9/2024 17:07:00,9/14/2024 14:30:00,1 +897,738,Shipped,M,9/9/2024 12:43:00,,9/11/2024 23:53:00,,1 +898,739,Processing,F,6/3/2024 8:44:00,,,,2 +899,741,Returned,F,10/9/2022 8:31:00,10/15/2022 10:57:00,10/10/2022 7:32:00,10/14/2022 12:18:00,1 +900,743,Complete,F,9/18/2023 1:29:00,,9/20/2023 9:55:00,9/24/2023 17:01:00,1 +901,743,SH,F,11/1/2023 1:29:00,,11/1/2023 6:55:00,,2 +902,744,Shipped,F,10/21/2021 9:28:00,,10/23/2021 2:11:00,,4 +903,746,Shipped,M,9/13/2022 12:33:00,,9/14/2022 23:40:00,,1 +904,746,Complete,M,5/22/2020 12:33:00,,5/23/2020 3:47:00,5/23/2020 14:44:00,2 +905,747,Cancelled,M,5/31/2022 17:18:00,,,,1 +906,748,Complete,M,1/7/2024 15:01:00,,1/9/2024 5:54:00,1/14/2024 1:19:00,1 +907,749,Complete,F,7/31/2024 6:58:00,,8/2/2024 17:58:00,8/6/2024 5:11:00,1 +908,750,Cancelled,M,5/27/2024 8:30:00,,,,1 +909,751,Complete,M,7/13/2021 10:11:00,,7/13/2021 14:27:00,7/16/2021 18:41:00,1 +910,752,Cancelled,F,9/8/2024 1:21:00,,,,1 +911,755,Shipped,M,7/4/2024 1:30:00,,7/4/2024 14:36:00,,1 +912,756,Complete,M,9/1/2024 16:03:00,,9/4/2024 3:48:00,9/7/2024 14:50:00,1 +913,758,Cancelled,M,1/26/2024 6:02:00,,,,2 +914,758,Shipped,M,1/9/2024 6:02:00,,1/10/2024 14:29:00,,1 +915,759,Processing,M,5/23/2023 4:51:00,,,,2 +916,760,Complete,F,8/19/2024 11:18:00,,8/20/2024 5:25:00,8/24/2024 6:09:00,1 +917,760,Processing,F,4/23/2024 11:18:00,,,,1 +918,760,Complete,F,7/2/2024 11:18:00,,7/4/2024 9:49:00,7/8/2024 10:36:00,1 +919,760,Processing,F,9/6/2024 11:18:00,,,,2 +920,761,Complete,M,7/2/2023 15:52:00,,7/5/2023 9:21:00,7/10/2023 0:53:00,2 +921,762,Complete,M,3/28/2023 8:26:00,,3/30/2023 0:20:00,4/2/2023 2:15:00,2 +922,763,Cancelled,M,7/18/2024 2:19:00,,,,1 +923,765,Complete,M,11/3/2022 12:05:00,,11/4/2022 2:00:00,11/5/2022 17:49:00,1 +924,765,Processing,M,8/5/2022 12:05:00,,,,1 +925,766,Shipped,F,9/24/2020 3:22:00,,9/27/2020 1:18:00,,1 +926,766,Shipped,F,9/23/2020 3:22:00,,9/23/2020 18:55:00,,3 +927,767,Processing,F,8/15/2024 9:54:00,,,,1 +928,768,Shipped,F,7/29/2023 3:34:00,,7/29/2023 23:47:00,,3 +929,768,Cancelled,F,1/3/2024 3:34:00,,,,4 +930,769,,M,6/15/2023 0:44:00,6/24/2023 3:35:00,6/17/2023 0:59:00,6/21/2023 13:15:00,2 +931,769,Returned,M,4/11/2023 0:44:00,4/16/2023 12:03:00,4/12/2023 23:35:00,4/14/2023 18:02:00,1 +932,769,Processing,M,5/11/2023 0:44:00,,,,1 +933,769,Shipped,M,8/31/2020 0:44:00,,9/1/2020 2:00:00,,2 +934,770,Processing,F,5/10/2022 11:08:00,,,,1 +935,772,Processing,M,5/26/2024 10:33:00,,,,1 +936,773,Shipped,M,9/7/2024 11:57:00,,9/9/2024 7:26:00,,2 +937,773,Cancelled,M,12/9/2023 11:57:00,,,,1 +938,773,Shipped,M,8/30/2023 11:57:00,,9/1/2023 14:02:00,,2 +939,774,Processing,F,3/16/2023 8:03:00,,,,2 +940,775,Complete,M,10/7/2023 17:15:00,,10/9/2023 20:34:00,10/11/2023 13:44:00,1 +941,775,Returned,M,8/20/2024 17:15:00,8/25/2024 19:50:00,8/21/2024 7:02:00,8/24/2024 23:33:00,1 +942,775,Cancelled,M,9/27/2023 17:15:00,,,,1 +943,775,Shipped,M,1/29/2024 17:15:00,,1/30/2024 0:04:00,,1 +944,777,Shipped,F,5/22/2022 7:47:00,,5/23/2022 3:12:00,,4 +945,780,Shipped,F,7/14/2024 11:05:00,,7/15/2024 14:21:00,,1 +946,784,Processing,M,7/7/2024 2:59:00,,,,1 +947,786,Complete,F,3/9/2022 12:47:00,,3/11/2022 21:54:00,3/16/2022 9:56:00,1 +948,786,Complete,F,11/15/2019 12:47:00,,11/16/2019 16:01:00,11/21/2019 8:33:00,3 +949,787,Shipped,M,7/16/2023 3:46:00,,7/16/2023 6:02:00,,1 +950,787,Complete,M,1/27/2024 3:46:00,,1/27/2024 11:07:00,1/31/2024 16:16:00,1 +951,788,Shipped,F,1/28/2023 17:37:00,,1/30/2023 1:29:00,,2 +952,789,Complete,F,5/20/2024 0:02:00,,5/22/2024 7:52:00,5/23/2024 6:50:00,1 +953,789,Shipped,F,2/20/2024 0:02:00,,2/20/2024 3:45:00,,3 +954,790,Returned,F,4/13/2023 15:08:00,4/19/2023 11:34:00,4/15/2023 6:09:00,4/17/2023 16:51:00,2 +955,790,Complete,F,10/15/2023 15:08:00,,10/15/2023 15:22:00,10/18/2023 14:47:00,1 +956,790,Complete,F,12/15/2022 15:08:00,,12/17/2022 22:06:00,12/18/2022 23:39:00,4 +957,790,Cancelled,F,11/9/2022 15:08:00,,,,1 +958,791,Shipped,M,4/17/2022 2:17:00,,4/19/2022 17:59:00,,1 +959,791,Returned,M,3/9/2020 2:17:00,3/17/2020 4:15:00,3/11/2020 0:57:00,3/15/2020 9:25:00,1 +960,791,Complete,M,8/27/2019 2:17:00,,8/27/2019 3:36:00,8/29/2019 19:04:00,1 +961,791,Shipped,M,9/26/2020 2:17:00,,9/28/2020 17:21:00,,1 +962,792,Returned,M,1/21/2023 3:17:00,1/27/2023 14:56:00,1/22/2023 12:43:00,1/25/2023 11:27:00,1 +963,792,Returned,M,8/8/2024 3:17:00,8/10/2024 17:10:00,8/8/2024 15:21:00,8/9/2024 7:29:00,1 +964,793,Processing,F,5/22/2024 10:35:00,,,,1 +965,793,Cancelled,F,6/27/2024 10:35:00,,,,2 +966,794,Returned,F,7/4/2024 10:38:00,7/8/2024 23:27:00,7/6/2024 1:53:00,7/8/2024 13:42:00,1 +967,795,Shipped,F,10/9/2022 8:59:00,,10/11/2022 10:39:00,,2 +968,795,Cancelled,F,7/29/2020 8:59:00,,,,1 +969,795,Cancelled,F,7/15/2023 8:59:00,,,,4 +970,795,Shipped,F,6/10/2022 8:59:00,,6/12/2022 7:21:00,,1 +971,796,Shipped,M,6/25/2023 17:53:00,,6/28/2023 4:55:00,,1 +972,796,Processing,M,7/26/2023 17:53:00,,,,1 +973,797,Cancelled,M,11/27/2022 0:54:00,,,,2 +974,798,Returned,F,10/12/2023 6:56:00,10/19/2023 0:13:00,10/12/2023 20:07:00,10/17/2023 19:09:00,1 +975,799,Returned,F,9/10/2024 15:06:53,9/14/2024 22:58:53,9/13/2024 1:24:53,9/14/2024 14:49:53,2 +976,800,Complete,M,2/18/2021 2:28:00,,2/20/2021 1:14:00,2/22/2021 11:15:00,3 +977,801,Complete,M,7/1/2023 3:12:00,,7/1/2023 20:45:00,7/3/2023 22:05:00,1 +978,802,Cancelled,M,8/2/2022 16:13:00,,,,2 +979,802,Shipped,M,7/7/2022 16:13:00,,7/7/2022 23:18:00,,1 +980,803,Processing,M,6/29/2023 18:41:00,,,,2 +981,804,Complete,M,1/22/2022 8:00:00,,1/24/2022 7:58:00,1/24/2022 18:20:00,1 +982,805,Processing,F,6/1/2024 6:53:00,,,,1 +983,805,Complete,F,6/13/2024 6:53:00,,6/13/2024 20:03:00,6/14/2024 14:01:00,1 +984,807,Complete,M,4/1/2023 7:01:00,,4/1/2023 19:19:00,4/4/2023 13:07:00,1 +985,808,Complete,F,12/29/2023 11:08:00,,12/31/2023 2:48:00,1/4/2024 20:43:00,1 +986,809,,F,6/5/2022 14:19:00,,,,1 +987,810,Shipped,F,9/6/2024 12:24:00,,9/7/2024 8:59:00,,1 +988,810,Processing,F,7/27/2024 12:24:00,,,,1 +989,811,Shipped,F,7/2/2023 11:08:00,,7/4/2023 21:30:00,,1 +990,812,Cancelled,M,6/15/2022 1:00:00,,,,1 +991,816,Processing,M,12/31/2020 18:12:00,,,,1 +992,816,Returned,M,6/10/2020 18:12:00,6/12/2020 9:11:00,6/11/2020 9:02:00,6/11/2020 13:22:00,1 +999,817,Shipped,,9/16/2022 16:48:00,,9/18/2022 11:59:00,,3 +994,818,Complete,F,10/14/2020 17:58:00,,10/17/2020 15:24:00,10/20/2020 20:42:00,1 +995,818,Processing,F,6/21/2021 17:58:00,,,,3 +996,819,Shipped,F,6/12/2023 14:41:00,,6/15/2023 14:20:00,,1 +997,820,Complete,F,3/20/2022 17:01:00,,3/23/2022 9:41:00,3/25/2022 19:46:00,2 +998,821,Shipped,M,9/1/2024 3:13:00,,9/3/2024 2:36:00,,1 +999,823,Processing,F,6/16/2023 16:48:00,,,,1 +1000,824,Complete,M,12/18/2022 11:45:00,,12/18/2022 19:36:00,12/19/2022 10:50:00,1 \ No newline at end of file diff --git a/dqops/sampledata/full_name_test.csv b/dqops/sampledata/full_name_test.csv index b9e0b22f5b..59687e3cce 100644 --- a/dqops/sampledata/full_name_test.csv +++ b/dqops/sampledata/full_name_test.csv @@ -18,4 +18,5 @@ id:INTEGER,full_name:STRING,result:INTEGER,date:LOCAL_DATE 17,Olszewski J@nusz,0,2022-03-11 18,KacprzakPatryk,0,2022-04-11 19,123Szymczak Leszek,0,2022-05-11 -20,L Dawid,0,2022-06-11 \ No newline at end of file +20,L Dawid,0,2022-06-11 +21,,,2022-06-11 \ No newline at end of file diff --git a/dqops/sampledata/geographic_coordinate_system_test.csv b/dqops/sampledata/geographic_coordinate_system_test.csv index a919d4dcfc..1034d4b8f2 100644 --- a/dqops/sampledata/geographic_coordinate_system_test.csv +++ b/dqops/sampledata/geographic_coordinate_system_test.csv @@ -18,4 +18,5 @@ id:INTEGER,gcs:DOUBLE,gcs_ok:INTEGER,date:LOCAL_DATE 17,-174.20336,2,2022-04-14 18,-134.88006,2,2022-04-14 19,360,0,2022-04-14 -20,-360,0,2022-04-14 \ No newline at end of file +20,-360,0,2022-04-14 +21,,,2022-04-14 \ No newline at end of file diff --git a/dqops/sampledata/ip4_test.csv b/dqops/sampledata/ip4_test.csv index 4b734cf89f..448a7c3c43 100644 --- a/dqops/sampledata/ip4_test.csv +++ b/dqops/sampledata/ip4_test.csv @@ -18,4 +18,5 @@ id:INTEGER,ip4:STRING,result:INTEGER,date:LOCAL_DATE 17,206212177195,0,2022-03-13 18,to nie jest IP4,0,2022-04-14 19,206-212-177-195,0,2022-05-15 -20,206.212.177,0,2022-06-16 \ No newline at end of file +20,206.212.177,0,2022-06-16 +21,,,2022-06-16 \ No newline at end of file diff --git a/dqops/sampledata/ip6_test.csv b/dqops/sampledata/ip6_test.csv index 55a12eb119..b2f44ed6e6 100644 --- a/dqops/sampledata/ip6_test.csv +++ b/dqops/sampledata/ip6_test.csv @@ -18,4 +18,5 @@ id:INTEGER,ip6:STRING,result:INTEGER,date:LOCAL_DATE 17,c2190b3f96f6da15bcac856add1a9e71,0,2022-03-13 18,to nie jest IP6,0,2022-04-14 19,c219.0b3f.96f6.da15.bcac.856a.dd1a.9e71,0,2022-05-15 -20,b972:ed68:6911:5212:0884:9395:387g:8ad5,0,2022-06-16 \ No newline at end of file +20,b972:ed68:6911:5212:0884:9395:387g:8ad5,0,2022-06-16 +21,,,2022-06-16 \ No newline at end of file diff --git a/dqops/sampledata/nulls_and_uniqueness.csv b/dqops/sampledata/nulls_and_uniqueness.csv index 554b5a88cc..27ffa19451 100644 --- a/dqops/sampledata/nulls_and_uniqueness.csv +++ b/dqops/sampledata/nulls_and_uniqueness.csv @@ -5,15 +5,15 @@ id:INTEGER,nulls:STRING,nulls_ok:INTEGER,unique_count:STRING,negative:INTEGER,us 4,Brunei,1,Slovenia,2,"=13135678943",1,21522,1,AB,0,PYG,1,2022-02-04,5,4 5,,0,Solomon Islands,4,"(1)5175413241",1,21536,1,BS,1,$,1,2022-02-05,5,5 6,Curacao,1,Somalia,-1,"1 586 456 2433",1,21550,1,IO,1,peso,0,2022-02-06,5,6 -7,,0,Thailand,-4," 16165240542 ",1,,0,,1,kr,0,2022-02-07,5,7 -8,Afghanistan,1,Timor-Leste,-45,"17345213489",1,215388888,1,,1,CUP,1,2022-02-08,5,8 +7,,0,Thailand,-4," 16165240542 ",1,,,,1,kr,0,2022-02-07,5,7 +8,Afghanistan,1,Timor-Leste,-45,"17345213489",1,215388888,0,,1,CUP,1,2022-02-08,5,8 9,Albania,1,United Arab Emirates,1,"+1 8105234567",1,21561,1,BD,0,zł,1,2022-02-09,5,9 10,Algeria,1,,1,"(906)6259999",1,86045,1,CM,0,dollar,0,2022-02-10,5,0 -11,Andorra,1,Zambia,156,"19472348976???",0,8604486044,1,CP,0,IRR,1,2022-02-11,5, +11,Andorra,1,Zambia,156,"19472348976???",0,8604486044,0,CP,0,IRR,1,2022-02-11,5, 12,Angola,1,Zimbabwe,78,"",0,86435,1,KY,1,KZT,1,2022-02-12,5, 13,,0,Kosovo,93,"16792345678",1,86024,1,,1,€,1,2022-02-13,5, 14,,0,Kuwait,20,"",0,12345-12345,0,TD,1,MZN,1,2022-02-14,5, -15,,0,,43,"12342456789#",0,3060830608,1,CO,0,SHP,1,2022-02-15,5, +15,,0,,43,"12342456789#",0,3060830608,0,CO,0,SHP,1,2022-02-15,5, 16,Ethiopia,1,Slovenia,-3,"=13261092976",0,30683,1,CO,0,real,0,2022-02-16,5, 17,,0,Solomon Islands,-83,"13305410987",1,31803,1,HR,1,¥,1,2022-02-17,5, 18,Hong Kong,1,Somalia,-22,"13805414567iowa",0,61914,1,CW,1,euro,0,2022-02-18,5, @@ -23,4 +23,5 @@ id:INTEGER,nulls:STRING,nulls_ok:INTEGER,unique_count:STRING,negative:INTEGER,us 22,,0,United Kingdom,3,"(+1)5671239999",1,66419,1,GA,1,shilling,0,2022-02-22,5,-22 23,Pakistan,1,,495,"(1)6141118766",1,22801-0001,1,,1,$,1,2022-02-23,5,-23 24,,0,Zimbabwe,91,"17400986784222",0,22803-0002,1,,1,pound,0,2022-02-24,5,-24 -25,,0,Kosovo,-1,"9372346785",1,2280722807,1,TZ,1,denar,0,2022-02-25,5,-25 \ No newline at end of file +25,,0,Kosovo,-1,"9372346785",1,2280722807,0,TZ,1,denar,0,2022-02-25,5,-25 +26,,,,,,,,,,,,,2022-02-25,, \ No newline at end of file diff --git a/dqops/sampledata/string_test_data.csv b/dqops/sampledata/string_test_data.csv index bdd706c890..d51e88b495 100644 --- a/dqops/sampledata/string_test_data.csv +++ b/dqops/sampledata/string_test_data.csv @@ -11,7 +11,7 @@ id:INTEGER,email:STRING,email_ok:INTEGER,surrounded_by_whitespace:STRING,surroun 10,allowed|character@mail.com,1,"Alaska",1,undefined,1,yes,1,Margarets Avenue,1,78,86045,2022-02-10,, 11,allowed~character@mail.com,1,"Arizona",1,married,0,no,1,Benton Harbor,1,,2345,2022-02-11,1, 12,dot.is.allowed.in.email@mail.com,1,"",1,blank,1,no,1,OneWord,1,89.6,86435,2022-02-12,, -13,_specialcharonbeginning@mail.com,1,"",1,married,0,no,1,Liberty Court,1,,86024,2022-02-13,1, +13,_specialcharonbeginning@mail.com,1," ",1,married,0,no,1,Liberty Court,1,,86024,2022-02-13,1, 14,specialcharonend_@mail.com,1,"Colorado",1,-,1,yes,1,,0,76,30601306,2022-02-14,, 15,invalid.single.char.top.domain@mail.a,0,"Connecticu",1,missing,1,y,1,,0,34,3060830,2022-02-15,0, 16,.dot.before.email.allowed@gmail.com,1,"Delaware",1,married,0,y,1,Branch Lane,1,21,30683,2022-02-16,0, @@ -28,4 +28,5 @@ id:INTEGER,email:STRING,email_ok:INTEGER,surrounded_by_whitespace:STRING,surroun 27,invalid.email.mail.com,0,"Louisiana",1,,1,1,1,,0,77,2345,2022-02-01,, 28,invalid@mail@mail.com,0,"Tennessee",1,married,0,f,1,Summerhouse St.,1,432.98,86435,2022-02-01,, 29,12345678901234567890123456789012345678901234567890123456789012345@toolong.localpart.com,0,"Texas",1,married,0,t,1,,0,34.12,86024,2022-02-01,, -30,domain.name.underscores.not.allowed@mail_box.com,0,"Ut ah",1,married,0,t,1,Hill Field Ave.,1,678.43,30601,2022-02-01,1, \ No newline at end of file +30,domain.name.underscores.not.allowed@mail_box.com,0,"Ut ah",1,married,0,t,1,Hill Field Ave.,1,678.43,30601,2022-02-01,1, +31,,,,,,,,,,,,,2022-02-01,, \ No newline at end of file diff --git a/dqops/sampledata/test_data_values_in_set.csv b/dqops/sampledata/test_data_values_in_set.csv index 477e37a0ad..a104b4dd0a 100644 --- a/dqops/sampledata/test_data_values_in_set.csv +++ b/dqops/sampledata/test_data_values_in_set.csv @@ -28,4 +28,5 @@ id:INTEGER,length_int:INTEGER,length_string:STRING,mix_string_int:STRING,strings 27,1234,zxc,33,b22b,2022-02-01,1,2022-02-01,2022-02-01,02/01/2022,01/02/2022,2022/02/01,"Feb 1, 2022",d3,,2022-02-27 28,12345,asd,bb,a111a,aa,0,2022-02-01,2022-02-01,02/01/2022,01/02/2022,2022/02/01,"Feb 1, 2022",d3,,2022-02-28 29,1234567,qwe,cc,d44d,2022-02-01,1,2022-02-01,2022-02-01,02/01/2022,01/02/2022,2022/02/01,"Feb 1, 2022",d3,,2022-03-01 -30,123,zxy,aa,a111a,22,0,2022-02-01,2022-02-01,02/01/2022,01/02/2022,2022/02/01,"Feb 1, 2022",d3,,2022-03-02 \ No newline at end of file +30,123,zxy,aa,a111a,22,0,2022-02-01,2022-02-01,02/01/2022,01/02/2022,2022/02/01,"Feb 1, 2022",d3,,2022-03-02 +31,,,,,,,2022-02-01,,,,,,,, \ No newline at end of file diff --git a/dqops/sampledata/uuid_test.csv b/dqops/sampledata/uuid_test.csv index 1849e467dd..905d08f321 100644 --- a/dqops/sampledata/uuid_test.csv +++ b/dqops/sampledata/uuid_test.csv @@ -16,6 +16,7 @@ id:INTEGER,uuid:STRING,result:INTEGER,date:LOCAL_DATE 15,26b5e2be-925b-11ed-a1eb-0242ac120002,1,2023-01-11 16,26x5e2be-925b-11ed-a1eb-0242ac120002,0,2023-02-12 17,26b5e2be-925b-112ed-a1eb-0242ac120002,0,2022-03-13 -18,26b5e2be-925b-11ed-@1eb-0242ac120002,0,2022-04-14 +18,26b5e2be-925b-12ed-@1eb-0242ac120002,0,2022-04-14 19,wrong UUID,0,2022-05-15 -20,2137,0,2022-06-16 \ No newline at end of file +20,2137,0,2022-06-16 +21,,,2022-06-16 \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 8fb03fa4cb..ed66010bf3 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -305,8 +306,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 24143a8781..aadc09197d 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -299,8 +300,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 96cf1dd65b..5e16075d0f 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -307,8 +308,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index d8e64d5292..2781f33ffe 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 894fb10c9a..0bc298b857 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -349,8 +350,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index c002815e54..7134854d4d 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 1658189b25..86f73acb18 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/acceptedvalues/AthenaColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 538d02b69e..eb5c83c086 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index b41610013e..12b64c14aa 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/bool/AthenaColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 347522c263..06faaaa108 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index e2908bb881..b4f428456d 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -264,8 +265,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index e14b8b0d24..8ae75ba292 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index 4d5aadc38a..b74f4f1b87 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/conversions/AthenaColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 7a80d8b547..21eba05598 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -143,7 +143,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index 38f8c44896..269d291246 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -171,7 +172,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -194,7 +195,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -230,7 +231,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -243,8 +244,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index d53c1a1e34..a8c829345c 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/customsql/AthenaColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -171,7 +172,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -194,7 +195,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -230,7 +231,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -243,8 +244,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datatype/AthenaColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datatype/AthenaColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 7fabc7fe13..06a2e0857e 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datatype/AthenaColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datatype/AthenaColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -43,7 +43,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -245,7 +247,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -265,7 +267,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -285,7 +287,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -304,16 +306,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index ac4011ed89..d056ef6347 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -286,8 +287,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index f4e9c69b5b..e52529dc8a 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 7439a8eac6..dcfeafa63c 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/datetime/AthenaColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -107,7 +108,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -121,7 +122,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,7 +136,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -163,7 +164,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -242,8 +243,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index c3cd228327..5352d760bc 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -276,8 +277,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index b024808e27..c73ca4d68d 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/integrity/AthenaColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -276,8 +277,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index c669fbe44d..5207aa8b24 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -136,12 +137,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -156,14 +157,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -189,7 +190,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -198,15 +199,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 940f17bb70..9612292d1c 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -74,7 +75,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -87,7 +88,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -100,7 +101,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -126,7 +127,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -137,12 +138,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -157,14 +158,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -199,15 +200,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index b689523ce4..7f79d95294 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -74,7 +75,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -87,7 +88,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -100,7 +101,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -126,7 +127,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -137,12 +138,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -157,14 +158,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -199,15 +200,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 3d294fdb76..b570fff8c5 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/nulls/AthenaColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -15,9 +15,10 @@ */ package com.dqops.athena.sensors.column.nulls; +import com.dqops.athena.BaseAthenaIntegrationTest; import com.dqops.checks.CheckTimeScale; import com.dqops.checks.column.checkspecs.nulls.ColumnNullsPercentCheckSpec; -import com.dqops.connectors.ProviderType; +import com.dqops.connectors.trino.AthenaConnectionSpecObjectMother; import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; import com.dqops.execution.sensors.SensorExecutionResult; import com.dqops.execution.sensors.SensorExecutionRunParameters; @@ -25,6 +26,7 @@ import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; import com.dqops.metadata.groupings.DataGroupingDimensionSource; import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.sources.ConnectionSpec; import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; @@ -32,9 +34,6 @@ import com.dqops.sampledata.SampleTableMetadata; import com.dqops.sampledata.SampleTableMetadataObjectMother; import com.dqops.sensors.column.nulls.ColumnNullsNullsPercentSensorParametersSpec; -import com.dqops.athena.BaseAthenaIntegrationTest; -import com.dqops.connectors.trino.AthenaConnectionSpecObjectMother; -import com.dqops.metadata.sources.ConnectionSpec; import com.dqops.testutils.ValueConverter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -42,9 +41,12 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; + +; @SpringBootTest public class AthenaColumnNullsNullsPercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -74,7 +76,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -87,7 +89,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -100,7 +102,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -126,7 +128,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -137,12 +139,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -157,14 +159,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -190,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -199,15 +201,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index bea6c9974f..dd3a25fc0b 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -249,8 +250,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 47fcb0cc7e..35222725c7 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -81,7 +82,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 1cace9792e..05c67095d2 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -81,7 +82,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericMeanSensorParametersSpecIntegrationTest.java index 979f5f4d6f..a1bd7843f5 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 5e2ab06cec..72afc61643 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 6b8b5c6a36..8bfce10837 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index b6625d3c43..5fa6c8fd77 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 178310a4c4..e48fff31c0 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 0caaa1cf02..cae5c7a2fe 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 657c698fe9..c0a4b3a7c9 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 32655f44c3..e1a2df64e3 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 2c0696c5b3..c3a4a95d26 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index a904f809c6..27da1c3307 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -249,8 +250,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericPercentileSensorParametersSpecIntegrationTest.java index cc57ccfbd8..c450c5cfd3 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -217,8 +218,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 09a4e22d88..36b901c2df 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index b4c7431d78..7365fd222b 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/numeric/AthenaColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 287fe38cf6..5a98eea1cc 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index 465f56e413..f4f8c00bda 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 8ba939c9eb..f3370f7f7b 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 78a375968f..5b2efe97d2 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index d07b2bb271..90c134d64e 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index ab7e87f653..48355db36f 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index 4bb9adaa19..720afb0a85 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,11 +225,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 6935b817c9..48ee757490 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,11 +225,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index d5071397d7..fe3b485723 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index bea4e59fb2..f6ff943c22 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 414a383505..7c99ed6234 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index a1d591ce9c..9693a34e3e 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 5910c42772..f87cf326cb 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index d93b96ee26..9b3bb685ea 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 57f5a7209d..16fd4d2f3f 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/patterns/AthenaColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 07f2d78025..50bee5d619 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 9f862ca6e3..ef323d15af 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -226,8 +227,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 53005618a2..486f6621d0 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -327,8 +328,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index f137919773..7ee9f7b4f2 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 80e15799f4..9b0aec4f8c 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/pii/AthenaColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index 12ba5eac39..44bfe6da45 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -276,8 +277,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 51960c5fae..b9a6f6f43d 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -254,8 +255,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 071f963394..57f03b7b3a 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index e46a75a41b..a886e1d2c1 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index b355729df5..23ad8dedad 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/text/AthenaColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -285,8 +286,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 59860d7345..7dbf1718e3 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index c0a1e89248..ce42b9b581 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 0a65546d3a..5b85195f86 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index 7d74b076d5..b28489e547 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/uniqueness/AthenaColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class AthenaColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index c4d76d94d6..c445330425 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -226,8 +227,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 1a24b10b20..cd90c6c764 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 4293820192..35c60a7864 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index eea3b74376..57838b0776 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -94,7 +95,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -107,7 +108,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -120,7 +121,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -146,7 +147,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 751e6d4d39..bd6ad11041 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -86,46 +86,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -138,14 +138,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 71ca3d654f..c295694d3f 100644 --- a/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/column/whitespace/AthenaColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -86,46 +86,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -138,14 +138,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/table/uniqueness/AthenaTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/table/uniqueness/AthenaTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..b904803b87 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/table/uniqueness/AthenaTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,289 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.athena.sensors.table.uniqueness; + +import com.dqops.athena.BaseAthenaIntegrationTest; +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.trino.AthenaConnectionSpecObjectMother; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.sources.ConnectionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class AthenaTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + private ConnectionSpec connectionSpec; + + @BeforeEach + void setUp() { + this.connectionSpec = AthenaConnectionSpecObjectMother.create(); + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, connectionSpec); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, connectionSpec); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/athena/sensors/table/uniqueness/AthenaTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/athena/sensors/table/uniqueness/AthenaTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..8e0c31adab --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/athena/sensors/table/uniqueness/AthenaTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,289 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.athena.sensors.table.uniqueness; + +import com.dqops.athena.BaseAthenaIntegrationTest; +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.trino.AthenaConnectionSpecObjectMother; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.sources.ConnectionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class AthenaTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseAthenaIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + private ConnectionSpec connectionSpec; + + @BeforeEach + void setUp() { + this.connectionSpec = AthenaConnectionSpecObjectMother.create(); + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, connectionSpec); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, connectionSpec); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 1013863609..4735e55bb6 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 18e4e6695a..83553a6c5f 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -296,8 +297,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index e667b6096c..95ef7eae71 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 69cacd0611..5311ebfa03 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -298,8 +299,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 32abf4e699..bf8a643a9e 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -345,8 +346,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index da0c7fd1b0..20ebbc795f 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 1511c3e497..5a0c87bb5e 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/acceptedvalues/BigQueryColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 8a4304f51c..55e520b4c1 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index cf6fad70a5..805502b1a5 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/bool/BigQueryColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 7bcae9be53..ae922c9beb 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index a72ab9d0fb..6526508198 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index e4c0f68f25..c137d0e357 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index 6482a9838c..7ad44b8c6b 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/conversions/BigQueryColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 4c19eeed37..b6e7b878b7 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index 7c11272801..a7358da8dd 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index 568bbfd5ac..e07dd55ae9 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/customsql/BigQueryColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datatype/BigQueryColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datatype/BigQueryColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 6d21660bca..5282352927 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datatype/BigQueryColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datatype/BigQueryColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +216,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +244,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +264,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +284,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +303,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Object::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index bc7da1b1aa..34853a3a7b 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index 5fab8db811..65dbd78661 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 7de54a9a50..a3d2f2ced9 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/datetime/BigQueryColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index f3f6640e54..d4380652cc 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 38b2294454..4b30612873 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/integrity/BigQueryColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index 7f7f3edd16..f0d8e3895c 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index f96f4e38f5..0816a0a3eb 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -17,7 +17,7 @@ import com.dqops.bigquery.BaseBigQueryIntegrationTest; import com.dqops.checks.CheckTimeScale; -import com.dqops.checks.column.checkspecs.nulls.ColumnNotNullsPercentCheckSpec;; +import com.dqops.checks.column.checkspecs.nulls.ColumnNotNullsPercentCheckSpec; import com.dqops.connectors.ProviderType; import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; import com.dqops.execution.sensors.SensorExecutionResult; @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 7594fe1d2d..7ca2cc6ff6 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 33c6849c9e..cc843fc1a2 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/nulls/BigQueryColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -17,7 +17,7 @@ import com.dqops.bigquery.BaseBigQueryIntegrationTest; import com.dqops.checks.CheckTimeScale; -import com.dqops.checks.column.checkspecs.nulls.ColumnNullsPercentCheckSpec;; +import com.dqops.checks.column.checkspecs.nulls.ColumnNullsPercentCheckSpec; import com.dqops.connectors.ProviderType; import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; import com.dqops.execution.sensors.SensorExecutionResult; @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNullsNullsPercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 7fc285f0c8..9570edb431 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 31a1e9b982..51422278e0 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 5c519b4691..631d07831c 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 993d7f1ca9..662095c7e1 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index ca41f5a2e1..2fd06fab9a 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index 67857c9c35..d383a59a35 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index cd405d00e4..ca8feb0e8a 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 8d6a18c8c0..c0344b3e0a 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 050a49ea8d..74cd738e29 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index f9cc2abf68..f24f7ef925 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 7b46b7b0d3..ea2ae65232 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index e1eb8781f4..dad220c151 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericPercentileSensorParametersSpecIntegrationTest.java index f2415cfd10..8011637195 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 1dd28a7b21..2f39783359 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 2a32e56e96..01106637d2 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/numeric/BigQueryColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 1e02afe4ef..eccc90a724 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index b62ff93e7f..2a430f1602 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 30296b80e2..582cee8cad 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 4cde4ead27..a7da8a4a58 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 76bcb189c3..53de2d00be 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 540f167261..ef2986a25f 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index e58bb74dbf..64196514bd 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 146c1c9849..5920b06804 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 0f02af118f..31f2a73e19 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index c170882088..b4ece5f23c 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index ab93901abe..833386eec6 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index eae0a42805..88927df9e3 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 81be5e24ef..01e2dc8b7a 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index ec60b50d28..a6b470dfc5 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index f6b59be271..a151ef36c5 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/patterns/BigQueryColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 99f660bb05..4f28ec2a2d 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 7cc7da7f30..ede048fa87 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index a251155ef6..31c523dd12 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -324,8 +325,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 543da151c7..0fe0246a2f 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index f95649eccd..512a88481c 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/pii/BigQueryColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index 34201d64cc..393ef10732 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 71accb4ab9..194508430a 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 56b84a3db6..5e172c3067 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 35fd90f7bb..b5da491b40 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index d2f31d2db0..36db2a28b7 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/text/BigQueryColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 80c9dea439..e81de0255e 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 9e47cb9d2d..d7af7001c9 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 3ec37d9b78..0db7281944 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index a8f141ff75..fa38bf7379 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/uniqueness/BigQueryColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class BigQueryColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index ede9b522b7..0a9426b6ce 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 30b89b31b1..21079a6cd7 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 9f276576b3..4ee50ec677 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 7cf136e7e5..5b071d324f 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index af28347b19..d61fcb7b38 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index b3aca28c0b..17155db366 100644 --- a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/column/whitespace/BigQueryColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/table/uniqueness/BigQueryTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/table/uniqueness/BigQueryTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..a8690ecbcc --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/table/uniqueness/BigQueryTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.bigquery.sensors.table.uniqueness; + +import com.dqops.bigquery.BaseBigQueryIntegrationTest; +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class BigQueryTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.bigquery); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.bigquery); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/bigquery/sensors/table/uniqueness/BigQueryTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/table/uniqueness/BigQueryTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..9b86c844d3 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/bigquery/sensors/table/uniqueness/BigQueryTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.bigquery.sensors.table.uniqueness; + +import com.dqops.bigquery.BaseBigQueryIntegrationTest; +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class BigQueryTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.bigquery); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.bigquery); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index a5678cde5d..48c2804259 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 5d540a7ea1..bef43a0150 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -296,8 +297,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index b5a060e452..73cbffce49 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 205c624361..df90d13ab1 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -306,8 +307,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 7637b3f982..7f5f2c9859 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -326,8 +327,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index cbcec63e1a..dfbbd53a69 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 0190b2da39..61699fb035 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/acceptedvalues/DatabricksColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index cdd13ea3b8..f7743df038 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 24ec57b7c2..25b0c6b116 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/bool/DatabricksColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 705a5f75bb..3ec7f48881 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 46ec24a516..2d43ede4c1 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index d5c6d023ec..526f69ab14 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index 287c8317ae..fff4988d4c 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/conversions/DatabricksColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 572d25a390..3ffdcd2b10 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index aa8339216d..c0a8f65d01 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index 09cc626a54..bb16b13564 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/customsql/DatabricksColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datatype/DatabricksColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datatype/DatabricksColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index d79ad53bae..00573e2329 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datatype/DatabricksColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datatype/DatabricksColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index 7fd4d71664..8f797fbce7 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index d77fc5b5f6..2634240f1b 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 2d14029193..6eae78f2f3 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/datetime/DatabricksColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -18,6 +18,7 @@ import com.dqops.checks.CheckTimeScale; import com.dqops.checks.column.checkspecs.datetime.ColumnDateValuesInFuturePercentCheckSpec; import com.dqops.connectors.ProviderType; +import com.dqops.databricks.BaseDatabricksIntegrationTest; import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; import com.dqops.execution.sensors.SensorExecutionResult; import com.dqops.execution.sensors.SensorExecutionRunParameters; @@ -32,7 +33,6 @@ import com.dqops.sampledata.SampleTableMetadata; import com.dqops.sampledata.SampleTableMetadataObjectMother; import com.dqops.sensors.column.datetime.ColumnDatetimeDateValuesInFuturePercentSensorParametersSpec; -import com.dqops.databricks.BaseDatabricksIntegrationTest; import com.dqops.testutils.ValueConverter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 4289b0e05c..54731a78b5 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 665bebd31f..c23cc3f374 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/integrity/DatabricksColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index 07b7f62d31..18dec8198d 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 8a9ad29a31..7489aa6eb2 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -40,9 +40,10 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 11f2427235..993c135d49 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 8e91867902..9fd5790449 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/nulls/DatabricksColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -40,9 +40,10 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNullsNullsPercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 0cc764004c..8fe385628a 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 60bf432e64..fad2759caf 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 4a625c32d8..9529eb3f49 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericMeanSensorParametersSpecIntegrationTest.java index 7c7d07b2e5..3fb0cfd088 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 60f13779ba..2e64c08850 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 74c9e0b373..f1a5282ebb 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index c0657253e7..0321c7370d 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 13b9b4ff41..f26ab60a90 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 3ea48cb73e..fcd6028355 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index a1dcd2327e..b47d8930d7 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index de3d361258..1f9ea4279a 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 3045df2064..e70453f321 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index 0baf664825..78c29c63b1 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericPercentileSensorParametersSpecIntegrationTest.java index 09b689093c..a5cc16c8ed 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index eabc766180..d19c1135f3 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index d6da228d72..c834e0e5f9 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/numeric/DatabricksColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index bc2cf009d1..d0862ee4cb 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index baa21c4351..52d4d658b7 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index a65898c8cf..88576d1296 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index d4111f2557..33802cc157 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 264dd9425f..f54e22b077 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 4dea131581..808eee8613 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index cd634b19fb..085b4f7a1b 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -15,10 +15,10 @@ */ package com.dqops.databricks.sensors.column.patterns; -import com.dqops.bigquery.BaseBigQueryIntegrationTest; import com.dqops.checks.CheckTimeScale; import com.dqops.checks.column.checkspecs.patterns.ColumnInvalidUsaZipcodeFoundCheckSpec; import com.dqops.connectors.ProviderType; +import com.dqops.databricks.BaseDatabricksIntegrationTest; import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; import com.dqops.execution.sensors.SensorExecutionResult; import com.dqops.execution.sensors.SensorExecutionRunParameters; @@ -43,9 +43,10 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest -public class DatabricksColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseBigQueryIntegrationTest { +public class DatabricksColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { private ColumnPatternsInvalidUsaZipcodeFormatFoundSensorParametersSpec sut; private UserHomeContext userHomeContext; private ColumnInvalidUsaZipcodeFoundCheckSpec checkSpec; @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 29fab56aff..b52fc3efe3 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index c176454db5..66bff82194 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index 4eabdb44dc..6a27fa57bc 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index b7cc44d91c..1f5ca2b156 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index f7e35a5df9..a88dca5865 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 14a3f6ab08..da58e8dda8 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index 264a99d80b..33b69fa83d 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 419f71f078..bc92044b70 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/patterns/DatabricksColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index ba38c7dab8..ba87432423 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 7257e6ad26..6e696a3886 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 79859a648e..d1ea883c5f 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -324,8 +325,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 81219784ac..5cff5f2e0e 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 0c84756c1f..697b77d714 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/pii/DatabricksColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index a1245c324e..160aebf6ef 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 61cab4c8e9..8fd6299ed6 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 1f398804e3..14ab5890b4 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index a826c82977..04bb45fd10 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 6458d7f945..b1d7ee2e98 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/text/DatabricksColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 5207077b36..22936c21b4 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 809fb55db0..ac8932faa5 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index ef35ffcd87..401df90de1 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index 2bf8ba41ed..0b32882465 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/uniqueness/DatabricksColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DatabricksColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index b307b8f317..44cff0185f 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 2d355d6435..4d4275d228 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 61d57fd6f3..9160603a81 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 7a3f14f6e1..da270bc170 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 4a78fe52c4..5c9f4cb579 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,14 +83,14 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "email_ok", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 702179f8e5..192ab071f4 100644 --- a/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/column/whitespace/DatabricksColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,14 +83,14 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "email_ok", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/table/uniqueness/DatabricksTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/table/uniqueness/DatabricksTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..496cf28551 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/table/uniqueness/DatabricksTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.databricks.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.databricks.BaseDatabricksIntegrationTest; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class DatabricksTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.databricks); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.databricks); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/databricks/sensors/table/uniqueness/DatabricksTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/databricks/sensors/table/uniqueness/DatabricksTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..33aa68c80f --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/databricks/sensors/table/uniqueness/DatabricksTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.databricks.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.databricks.BaseDatabricksIntegrationTest; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class DatabricksTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseDatabricksIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.databricks); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.databricks); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 9843d1f4ce..92e007421d 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index ebed77b9a6..220cad8c42 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -298,8 +299,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 496baffaf0..82dca36443 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -307,8 +308,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 79cb0d328d..6fdc064dba 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -287,8 +288,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index ed7814c507..7533e2b3f5 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -329,8 +330,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index 1d66efe033..382341df71 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index d1749a683f..e01ab950b2 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/acceptedvalues/DuckdbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index df8e375c95..d055ee5abc 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 4962500dfb..81e73e5954 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/bool/DuckdbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index d69bbb001d..34f5b1cf92 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 40a40d3f87..8479980818 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -264,8 +265,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index b0dff7fa26..7065657492 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -226,8 +227,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index 46439d4217..d05f53f6fd 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/conversions/DuckdbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -227,8 +228,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index ea1188b134..2090fccecd 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -143,7 +143,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index ffe2a4796b..60771d3996 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -171,7 +172,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -194,7 +195,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -230,7 +231,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -243,8 +244,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index 7b31394b14..adcbf38f27 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/customsql/DuckdbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -170,7 +171,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -193,7 +194,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -229,7 +230,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -242,8 +243,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datatype/DuckdbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datatype/DuckdbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 09f241f051..14ed23d363 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datatype/DuckdbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datatype/DuckdbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -43,7 +43,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,7 +220,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -246,7 +248,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -266,7 +268,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -286,7 +288,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -305,16 +307,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index 6d73840692..9a77a07472 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -286,8 +287,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index ed23758793..11909f6230 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 611da61601..d2018236e2 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/datetime/DuckdbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -108,7 +109,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -122,7 +123,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -136,7 +137,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -164,7 +165,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -243,8 +244,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index f1e09a362d..a721a290d1 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -278,8 +279,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index a9e375b352..7c9970e1b3 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/integrity/DuckdbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -278,8 +279,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index ef32a79270..d674afd9c2 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -137,13 +138,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); - + Assertions.assertEquals(11, nullValues.length); } @Test @@ -157,14 +157,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -190,7 +190,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -199,15 +199,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 7b337b5f02..879b9fcd97 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -75,7 +76,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -88,7 +89,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -101,7 +102,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -127,7 +128,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -138,12 +139,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -158,14 +159,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -200,15 +201,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 2b8166813e..8c64e8c6ff 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -74,7 +75,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -87,7 +88,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -100,7 +101,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -126,7 +127,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -137,12 +138,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -157,14 +158,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -199,15 +200,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index b62b90beee..2c4802a622 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/nulls/DuckdbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -75,7 +76,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -88,7 +89,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -101,7 +102,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -127,7 +128,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -138,12 +139,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -158,14 +159,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -200,15 +201,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index b1ee32274a..58b23fa659 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -249,8 +250,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 5340e8951d..02eab9b16e 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -81,7 +82,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 74e571c13a..7d86ade4ca 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -81,7 +82,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index dd7b058ebe..e7cc24f686 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnNumericNegativeCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index bb77264edd..519a579d90 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index f6ffd44058..b8a178fb96 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 93e51784d7..234ed77fd3 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index b671b42911..ff7a598cb4 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index ac771b875a..77ccc64504 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 21a507333e..20ff90b304 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index fa436206bd..e3400c2771 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index fe4d944f11..cbd442c965 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -249,8 +250,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericPercentileSensorParametersSpecIntegrationTest.java index f9f10693c3..d83152844d 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 87d11e84b5..88fa570917 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/numeric/DuckdbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index b245a3dfde..6ebd27575d 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index 280c88a1b9..ec8c3ea41b 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 3d4552b17e..59a5a4b17d 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index a6cd5b48eb..b0e162de08 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index f5648481f6..92c2a4dbf6 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 161f73f947..f5c38b6122 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index a7f1668cb8..86a2676d0d 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -224,11 +225,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index a595a2e72b..0419074d84 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -224,11 +225,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 9ca4219650..a8c99333cb 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index e17b327de2..1c2a7833b9 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 72bd1a7730..2e2c9f023c 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 386db47808..0945a8f2a9 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 91128865b2..febc4b72ec 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index 747c609b63..9deb848c1b 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 18135dea78..40bebb56a8 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/patterns/DuckdbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index f32cdf5de5..00378ff059 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 1d548e5e58..cc50791a6a 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 8ff8927d84..65f8ca5d2a 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest class DuckdbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -325,8 +326,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 2cfee01858..04e9355bb5 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 7f85b77fe3..1e77764430 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/pii/DuckdbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index e741b22f5d..6679093676 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 6dd043b1b0..9fabda749f 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 3e100ee931..43155a1836 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 53aa28d348..fdbb2b23ae 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index f2a38b47ce..2f88e1d1f2 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/text/DuckdbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -285,8 +286,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index d351b419b8..6af1adafba 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index ac071c1142..22fb36868d 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 7d054290b2..bde8101bd7 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index 5716c3201f..edd3e1fa83 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/uniqueness/DuckdbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class DuckdbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 2d28092aae..49460c3ce8 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 60b673fdab..fe656f6aef 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -226,8 +227,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 22f5edd88a..9cf00b93a0 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index eec6e32f0f..07f23e7da6 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -95,7 +96,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -108,7 +109,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -121,7 +122,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -147,7 +148,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -226,8 +227,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 7fa3cb8fd5..211da68b10 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -93,7 +93,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -106,7 +106,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -119,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -145,7 +145,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 032758dd41..1b732c1b13 100644 --- a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/column/whitespace/DuckdbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -93,7 +93,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -106,7 +106,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -119,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -145,7 +145,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/uniqueness/DuckdbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/uniqueness/DuckdbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..ca5b6499e0 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/uniqueness/DuckdbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,289 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.duckdb.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother; +import com.dqops.connectors.duckdb.config.DuckdbFilesFormatType; +import com.dqops.duckdb.BaseDuckdbIntegrationTest; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.sources.ConnectionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class DuckdbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbFilesFormatType.csv); + String csvFileName = SampleCsvFileNames.test_data_values_in_set; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile( + csvFileName, connectionSpec); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbFilesFormatType.csv); + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile( + csvFileName, connectionSpec); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/uniqueness/DuckdbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/uniqueness/DuckdbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..46992a5da0 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/duckdb/sensors/table/uniqueness/DuckdbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,289 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.duckdb.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.duckdb.DuckdbConnectionSpecObjectMother; +import com.dqops.connectors.duckdb.config.DuckdbFilesFormatType; +import com.dqops.duckdb.BaseDuckdbIntegrationTest; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.sources.ConnectionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class DuckdbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseDuckdbIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbFilesFormatType.csv); + String csvFileName = SampleCsvFileNames.test_data_values_in_set; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile( + csvFileName, connectionSpec); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + ConnectionSpec connectionSpec = DuckdbConnectionSpecObjectMother.createForFiles(DuckdbFilesFormatType.csv); + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForExplicitCsvFile( + csvFileName, connectionSpec); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index df4bdf191c..5f0567359e 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 83602ae617..ad251da4b9 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -295,8 +296,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 1322c90c06..8e0aa12fb2 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index da4612213c..884dcc28c1 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -298,8 +299,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 3b759acf1f..591f0f4e8d 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -346,8 +347,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index f6117a3a4d..48f5a73113 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 4367f47ea1..577fe1f310 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/acceptedvalues/MysqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 2b354600b4..506c2e7109 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 3310f390b9..1e7e1982d8 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/bool/MysqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 6bd19069f8..a7a08ab6b1 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 71e52b5aa6..9b78139c98 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index a295a30cb5..a8d8e5f8e3 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index d3453fe12f..d6cfd6a80e 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/conversions/MysqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 30a3548528..38b9ef9572 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index d09beb7b3a..1addbf5c2c 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index a7d28656f2..8e29e3282b 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/customsql/MysqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datatype/MysqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datatype/MysqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 3d3b2f5fa2..a1bcb76daf 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datatype/MysqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datatype/MysqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -216,7 +218,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -244,7 +246,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -264,7 +266,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -284,7 +286,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -303,16 +305,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index ac90e683ba..6692188634 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index 186f35aae5..a5150a7a16 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index e5b065a85d..4fa12bc2f9 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/datetime/MysqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index d4f7e48c39..28e764975f 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -274,8 +275,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index d77dd26854..23d57ef5f8 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/integrity/MysqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index 6abe20a319..7edea7f7c0 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 28c6488c5c..3df3faa0bd 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 1bd49cf2c7..d3e37225db 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -70,7 +71,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14.0, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -83,7 +84,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14.0, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -96,7 +97,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14.0, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -122,7 +123,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -133,12 +134,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -153,14 +154,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -186,7 +187,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -195,15 +196,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index ece7d69a4c..19d869b89e 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/nulls/MysqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 4965526fb4..976488b2e6 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 78c28b014c..70d33381e8 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index ea9c350fc4..4d99f86540 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericMeanSensorParametersSpecIntegrationTest.java index 0b3da5f604..6a0de9dbac 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index ea918caa24..d38b23a76a 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index d2ee873fa4..54b6230c31 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index c0116c0e0e..543aae288f 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index c4acb9514a..07884597de 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 86985344d4..62435b1874 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 6f1f7bfcda..84e634959b 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index ac2ef9fc38..1056def6d3 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 09d6a203ba..0b69d284d1 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index 0ab68a3752..6aa8ec99c5 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 660de9c94f..a4f6249c37 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 3f591ef025..c4371b005e 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/numeric/MysqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 69c7f72a06..ffdbd7b8aa 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index 1cc1d98b1e..ee979447d6 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 841bda8bae..d0b8d44ec8 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 0f45284e09..3e4e4ffe82 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 1ab1308059..3b95a2eca2 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 8d6da10bb6..c48ddb328b 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index 9eca361c0c..7fcb5998fa 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 587c0d264c..941934d04e 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 851c8b9cfd..1402826a7d 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index da15bb41a4..b665eddaf3 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 4c4226b67d..87096260c1 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 55ac3690bb..f10d6c9a7a 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 39c7a6d963..c16aa1c7b2 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index d8eed206b8..a8a821b1f3 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 855cd28d67..7fd5313b95 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/patterns/MysqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index c10bfc8a4e..9f11f2faa7 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index e86996019f..049c1186c1 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 7abe201d26..89c54f5bc0 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest class MysqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -323,8 +324,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index ca80065c26..bbeb41374d 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index dd3d6abe45..122ac5aefa 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/pii/MysqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index ac48a87dfe..ee268f841b 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 9add8e2ae3..5c51e25f5d 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 0427ea9c6f..f0c9824e70 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 39a47d6ca2..b9126796b6 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 117521f084..8c5b35b2f2 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/text/MysqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index ef02edea83..3967aaf81c 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 18a9a739d5..41cf7d9fb9 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index b536a7abda..0c65862789 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index be2bbb8508..feab20beec 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/uniqueness/MysqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class MysqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index ad3fec4484..942a4f6f10 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index b4e9a8b11a..517f6cfc22 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index f95bb05847..4714b81453 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 2313eace78..55c44c2aaf 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index e07b6f08de..58642d9c85 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 794304d0a2..ad2b0e3664 100644 --- a/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/column/whitespace/MysqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/table/uniqueness/MysqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/table/uniqueness/MysqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..febfa93673 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/table/uniqueness/MysqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.mysql.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.mysql.BaseMysqlIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class MysqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.mysql); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.mysql); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/mysql/sensors/table/uniqueness/MysqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/mysql/sensors/table/uniqueness/MysqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..7f197ed3b2 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/mysql/sensors/table/uniqueness/MysqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.mysql.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.mysql.BaseMysqlIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class MysqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseMysqlIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.mysql); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.mysql); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 98c421e9a8..66e9b9de1f 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 8c7053f63e..d64930f8dd 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -295,8 +296,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index a1afdec82b..0302fc372d 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index ef03b8cd86..d3593be699 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -298,8 +299,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 4232815a57..e7822c0a96 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -345,8 +346,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index 15b87c7af6..db404d95ed 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 6e58824b26..4c761e61cb 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/acceptedvalues/OracleColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 277a735a77..57ae92965e 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("1")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index df85f889e5..5bce7bc702 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/bool/OracleColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("0")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index a804fb5238..f556520f6c 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 9635421b15..c190afaed2 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -262,8 +263,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index a1446cc832..7825484beb 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index 5dc2a5d6fa..b1bf64bfcd 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/conversions/OracleColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 001f0de7ce..bc0268e2ae 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -141,7 +141,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index be2c055bc5..ed422161ab 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index 350f1372f9..f4a29e078b 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/customsql/OracleColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datatype/OracleColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datatype/OracleColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 31498765a9..f3f4455128 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datatype/OracleColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datatype/OracleColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index 2c276704ef..de8af83af7 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -284,8 +285,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index 2217dabb6e..ab947cd7ae 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 0c306f78d7..f51cd20e99 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/datetime/OracleColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -1,7 +1,6 @@ package com.dqops.oracle.sensors.column.datetime; - import com.dqops.checks.CheckTimeScale; import com.dqops.checks.column.checkspecs.datetime.ColumnDateValuesInFuturePercentCheckSpec; import com.dqops.connectors.ProviderType; @@ -35,6 +34,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -91,7 +91,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -105,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -119,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -147,7 +147,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -226,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.get(0).startsWith("2999")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index c0a9404732..fc4cd30b13 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index e84ad1bdc2..56a3225dc1 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index c10513d1a1..c8eabb7d7a 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index b2fbef7913..b8b3a72b53 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -40,9 +40,10 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 91cd59d849..6377f92e98 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(14, (double) resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(14, (double) resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(14, (double) resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(14, (double) resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 8a17f885e5..b773e8ee45 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/nulls/OracleColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -40,9 +40,10 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNullsNullsPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 93977c17ca..a6ae4e560a 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 76d384ca25..15fa06566a 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index ff13052c87..4e4a67bbb6 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericMeanSensorParametersSpecIntegrationTest.java index cf45810c32..6000b4b83e 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 37e3721da1..fc54365f8b 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 7adfda9127..2638a7419d 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index da989b1090..d4fd9a54c2 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 959a19f477..a8b53d1295 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 33d221e8af..6eb10deadb 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 208ba28c46..19626e77d3 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 99eb916edf..db723a8e1b 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index e90eb780e5..a4accbd55f 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index d6b31ca26d..660edceffb 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericPercentileSensorParametersSpecIntegrationTest.java index 281e65e288..08b1a80c10 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 4d6b9ae0d5..a1003d81f3 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index a91d56ec2a..98731bc81e 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/numeric/OracleColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 050ebe19ea..a07927ac0f 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index fd8d442d30..7aeea0f19b 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 45f2d659c7..517df36e50 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index d069bfb914..3a9a3d2b89 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index ecc979501c..837f98e3b4 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 7190632175..c5d192a86a 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index 13988cd74b..4b4437bccd 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 288ff27125..779acf11b5 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index fcf6b06ef9..72615bb0cc 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index 1309b5218f..7e26d84c3b 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 033ee9a78b..13c9104e3e 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 25b3377caf..fcccf4ef38 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index be5a5ba382..36e77bee47 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index c147a014f8..ca4f4a8fab 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 51d277608d..44337b94ac 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/patterns/OracleColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 625ad3e311..0975ff4159 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 00bc6874cf..aba4af4c6c 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 1512b6083b..f7304294e0 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -308,8 +309,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 62f2f31454..dc1f18d50c 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 6fbf7be85d..3b2ecc3c70 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/pii/OracleColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index 99b9275883..a595211bf2 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index dbb9efaa0b..34bf01baa0 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index e0ed061de1..f5d6614f6a 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index df61d3778b..f9ac8004a8 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index b4bcbf2607..d1fddffd8b 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/text/OracleColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 4b440aee6d..d37fe738a3 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 7fcb865578..baa41c967f 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index c9d833a676..0b9e45cd36 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index 31abc7c961..4f144aefe5 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/uniqueness/OracleColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class OracleColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 969220fe29..4f59a47a93 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 46a04b8764..b30ebed914 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 237849e747..d2ed5f9c56 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index f57e3238a1..d5361c9c5f 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 3d2fc9eef2..0c43061520 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 25fc9e4a76..730b590683 100644 --- a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/whitespace/OracleColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/table/uniqueness/OracleTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/table/uniqueness/OracleTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..ea0f3b1766 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/table/uniqueness/OracleTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class OracleTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.oracle); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/table/uniqueness/OracleTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/table/uniqueness/OracleTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..71d9ef9577 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/table/uniqueness/OracleTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class OracleTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.oracle); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 8d2670a5cd..6e46b6d8e4 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 673f446188..283a25830d 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -295,8 +296,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 6e716202f4..30c6fd7190 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index b3c2db5080..7ee6899d06 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -300,8 +301,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index b970857d8f..1cdb3e39f6 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -326,8 +327,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index 6e33c171cf..406cd72cfa 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index c4e55a3b32..9d736d15a7 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/acceptedvalues/PostgresqlColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 10d416f153..e7fffa19ea 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -219,8 +220,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 2171dd0441..71127196e1 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/bool/PostgresqlColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 10a7ea5f35..4c0f278014 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index f482c0cf05..d5e516b274 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index fd2fa81cbf..24b65caf2c 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index d97fff87c4..07e89847bf 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/conversions/PostgresqlColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 43df41141e..0a07ba13d5 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index 12dbb3c725..1bebc80dcb 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index cdc93a7775..522f583ee2 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/customsql/PostgresqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datatype/PostgresqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datatype/PostgresqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 2fb8b8893b..da23605f94 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datatype/PostgresqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datatype/PostgresqlColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index d6aa21181d..3721665297 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index bf244212b7..027a242137 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index d09c2ab299..9b6ae60180 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/datetime/PostgresqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 78682815dd..8070482f9f 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -270,8 +271,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index b22329e08f..39e76ae63e 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/integrity/PostgresqlColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -270,8 +271,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index b25c8c25be..61b7b14aef 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index d66662df6a..7d265fd11e 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index c73955150c..6a5640f4de 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 9c1faccf77..200cbfea65 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/nulls/PostgresqlColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 913a443557..167153c2ad 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index fcf72294a8..4c82d105ab 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -217,8 +218,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index fe541f2ab1..3e539ba555 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericMeanSensorParametersSpecIntegrationTest.java index 7bf909d249..2055314d51 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index c051072bec..465fde38bf 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 6a86697c79..aeb1ed0911 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index cfff0f516a..d81d7624df 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index a4bcdc3793..df5aea86f7 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 770aa7c823..a200e79b2c 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index d357347464..64443a94e0 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 7e3352705a..8d02b7d914 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 5e1199ca4c..30ac32df22 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index 909d74f84e..f2c3962075 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericPercentileSensorParametersSpecIntegrationTest.java index 9840488522..b4916cc1ad 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index a48bebb3f0..7c545ea02a 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 1810f4db88..6d831febbb 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/numeric/PostgresqlColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 7c4e923230..1a5b3b7794 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index 9b371d5d40..13998e4f6f 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 0863332ef3..d1c754982b 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 7f72dbd907..7758ab2000 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 7887c2e73e..95e5cb2a0f 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index ea1be70d85..0b4d4d6462 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index 89ec9d2e4b..fb1b2de8f3 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index f426fa4b76..b34ee704f2 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 12cd7df6ea..cbe95dcf22 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index c262c8925d..f82629800c 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 076b45d2ac..08adb89ba0 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 9135c98772..00054098e9 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 6ae541ed03..0b85c69f3a 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index af591f91a4..0f678cc953 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 3793ba5829..bfa8019e21 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/patterns/PostgresqlColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 49744d0aec..fba1c8342e 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 6e25cfe572..2c515e9fdb 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index ee40245425..ef45ced49f 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest class PostgresqlColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -323,8 +324,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 0f406d6a68..589bf02a75 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 9626ba1199..c7868b78f7 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/pii/PostgresqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index 6c1f020e5a..eb979cd01c 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index e91977b45a..5f6e73a952 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 941bc9afa8..e145bbb20a 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 73a43c7369..7c7f446834 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 8a4c617b47..4ff45c3d7e 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/text/PostgresqlColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index f2e56f958a..25c6f1615f 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 4dd67c05b3..24813b64b9 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index fccf14e64b..6b61f2223a 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index ea5831e865..596d9af6c2 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/uniqueness/PostgresqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PostgresqlColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 3f0c294549..2cefbc3799 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 8fa7b4c03b..29561c9207 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 1056fcb3aa..729a250a73 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 0a9ce39c81..1be6809bde 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 2bac186c21..28571eae9b 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -90,7 +90,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index abb51470a7..0b40bc6c79 100644 --- a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/column/whitespace/PostgresqlColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -90,7 +90,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/table/uniqueness/PostgresqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/table/uniqueness/PostgresqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..b21ec9cbbc --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/table/uniqueness/PostgresqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.postgresql.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.postgresql.BasePostgresqlIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class PostgresqlTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.postgresql); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.postgresql); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/postgresql/sensors/table/uniqueness/PostgresqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/table/uniqueness/PostgresqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..2808c19ea7 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/postgresql/sensors/table/uniqueness/PostgresqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.postgresql.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.postgresql.BasePostgresqlIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class PostgresqlTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BasePostgresqlIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.postgresql); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.postgresql); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index f6a81e7901..62a386044d 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 3caafa0848..4732d0a032 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -296,8 +297,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index ea19a15dca..0995c0979a 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 0fd252f53a..72ccfe8af4 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -298,8 +299,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 261e8cd415..a721853e1d 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -345,8 +346,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index 5eeefb2dce..da16a74b4e 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 1246e78e65..cee611daff 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/acceptedvalues/PrestoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 12e7ff2bef..e63317d5f2 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 07483b69be..b37cff392a 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/bool/PrestoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 2b1b6fd2bd..ca10287ef3 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index ecdd73c1a2..4001292d9b 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index 0ac5db411c..de2dc5b634 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index a42d5f886b..17b6a0c2bd 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/conversions/PrestoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 0f06c026f9..18ad99e72e 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index 2934eddb20..e2311d5c63 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index ee354a7132..c1fc846c50 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/customsql/PrestoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datatype/PrestoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datatype/PrestoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 8e71456e6a..e5f5141f69 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datatype/PrestoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datatype/PrestoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index 386687488d..b016c67bcb 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index 631e57ce2e..f165ad640c 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 379e40750f..541543250f 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/datetime/PrestoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -15,10 +15,6 @@ */ package com.dqops.presto.sensors.column.datetime; -import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; -import com.dqops.metadata.groupings.DataGroupingDimensionSource; -import com.dqops.metadata.groupings.DataGroupingDimensionSpec; -import com.dqops.presto.BasePrestoIntegrationTest; import com.dqops.checks.CheckTimeScale; import com.dqops.checks.column.checkspecs.datetime.ColumnDateValuesInFuturePercentCheckSpec; import com.dqops.connectors.ProviderType; @@ -26,8 +22,12 @@ import com.dqops.execution.sensors.SensorExecutionResult; import com.dqops.execution.sensors.SensorExecutionRunParameters; import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.presto.BasePrestoIntegrationTest; import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; import com.dqops.sampledata.SampleCsvFileNames; import com.dqops.sampledata.SampleTableMetadata; @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 51dbce9b05..a74528c9e6 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 6526b1e48e..45b7b5b375 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/integrity/PrestoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index b1301d7bbd..88eca98932 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 1273ca0328..8912cd6192 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 53d191da11..26b317a3b4 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index a6219b386f..dc7ef7bbfc 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/nulls/PrestoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNullsNullsPercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index f1321c6396..2bccb44c8a 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 8cfa3a5368..f070a866d9 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index cbb8e626dd..36d9d41866 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericMeanSensorParametersSpecIntegrationTest.java index 6d68b5782d..6e7fafc138 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index cafa45b3f4..69f956a6f0 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index f539a751ec..cddb253194 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index 614c817e95..41dc65ace5 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 71ca317a36..168e3cc3fd 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 066f80114f..55bdfaa9e7 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index cfc2208370..b126371936 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 8439eedeaa..7d34e5f7b8 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index d9891c7ac9..37f3f813f8 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index f0b0f15d48..fd395fb6d9 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericPercentileSensorParametersSpecIntegrationTest.java index f0b7a80fce..08cbe1844d 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 035a4e43b2..4eefb7b164 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index c47e016ba1..5a303f7813 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/numeric/PrestoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 42c72d249c..2e8056de12 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index 02af63c4c5..a93f9fa085 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index f43d4e5aa2..303265c66d 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 0a605faf7c..f8872da6bb 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 6a5e9f7f93..562669107f 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 667be543e4..f79ad7459f 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index 2666041b29..fb146a3464 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 4fbf304085..0d589fa034 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 2a639c87b1..6d6e323a25 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index f6513b8e77..7986cec008 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index a293467f8a..ff8377c703 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 1ad27ff8c3..e2635f2c6b 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index eb97c75aac..744085f505 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index b0b6f8b5df..921cb328c2 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 93f3a1d7af..46de71a6ba 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/patterns/PrestoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index a6e83058b9..474776fee7 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index e1f360a6af..8b85c0b563 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 7ef931cdb4..d21fa2c546 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -324,8 +325,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index cf948f3960..289cede00e 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index a97c9c85fc..da11bc95d1 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/pii/PrestoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index 524bf78192..ed391f85dd 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index c66c65e006..7be1c3584f 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 94fa2545bb..964c998747 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index bf1148567f..7cf7dd0ea8 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index ade500bfdb..b085e33514 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/text/PrestoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 05bf902fa6..92707bf68d 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 9391763711..9fea15148e 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 018fea06e0..ab50c2aeab 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index d1ac98a2bc..f6269ec208 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/uniqueness/PrestoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class PrestoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 32058bb15c..b01ee8c574 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index cef7c967fd..f4121f2475 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 3771842d91..8294b59f60 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 563605757a..a489be9448 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index f8a37cf2ab..4305d165a4 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index fa00b5f3ea..cbc433a7da 100644 --- a/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/column/whitespace/PrestoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/table/uniqueness/PrestoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/table/uniqueness/PrestoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..81d8ea1aef --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/table/uniqueness/PrestoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.presto.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.presto.BasePrestoIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class PrestoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.presto); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.presto); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/presto/sensors/table/uniqueness/PrestoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/presto/sensors/table/uniqueness/PrestoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..dd2edd14b6 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/presto/sensors/table/uniqueness/PrestoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.presto.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.presto.BasePrestoIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class PrestoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BasePrestoIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.presto); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.presto); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index b78a76d522..fe76d8810c 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 4d228e4659..032a904fd2 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -295,8 +296,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index d5442d0de4..78c0185cf8 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index c67fb208df..1b5e799392 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -300,8 +301,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index bb2636db38..7dc5a50cd2 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -326,8 +327,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index 71b2cee346..4735f610f3 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 9739f4df12..15793139b4 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/acceptedvalues/RedshiftColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index a4a08d40eb..8aeabaa2d3 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 04b4cbe811..678b23330e 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/bool/RedshiftColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 2c0c6d1bb0..9b07f37039 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 1b682df401..5f769e325f 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index 5c19275fd1..60cfca4fc9 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index f8093539d5..491c60442f 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/conversions/RedshiftColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index e66c5f6444..b6062f5fd2 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index 9c21e00e93..0a5059356c 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index a4b0fac295..dbeb47817a 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/customsql/RedshiftColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datatype/RedshiftColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datatype/RedshiftColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 4198fcc338..1742b57acf 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datatype/RedshiftColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datatype/RedshiftColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index e2caae2e07..58e3fcf940 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index 1dd04d1db3..55d7d17af9 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 25bf41faa4..81e70e15b0 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/datetime/RedshiftColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 2a903c73d7..ad4a2cc943 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -270,8 +271,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index edf5b5200f..62b275d84d 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/integrity/RedshiftColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -270,8 +271,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index cf2646e638..43f739eec4 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index ab0e84a58f..f1608863d0 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 3d6cb39a83..7e430b0c03 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index c738aeb7ae..e07b0b8b9b 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/nulls/RedshiftColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index c9f4535592..67f847041c 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 7c30f773da..9f3346bcbd 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 80905a3ba7..3a0862d508 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericMeanSensorParametersSpecIntegrationTest.java index 4bbcf0af32..ce17e18519 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 324cd026db..e7ad34786a 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 7f488dfba7..eff8dc3021 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index 73350e3180..e80adb5ff0 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index f5250a1e39..60f46e6603 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index e3553f939c..653acd7108 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 396fd20d46..1f16b35c91 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 7489f8a470..a6f2e03b48 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 70dcfc47e8..2d6105660a 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index d543ebafb4..7788bc33c4 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericPercentileSensorParametersSpecIntegrationTest.java index c4a94479cc..f21288e98e 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 18e285d225..2718b9d918 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index bfc68543a7..b68378792d 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/numeric/RedshiftColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index ba448d1223..4de6721251 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index cc82a40f03..a4f536a1e0 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 1bf48c00d3..513b5e1c3d 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 2975b6fc3f..577af45f58 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 9f214af368..a12fd257ab 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 126383a5ba..43f7a04caa 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index 40807cc348..bb73786d6c 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 8d60e4853d..65ac5c1d81 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 2d8e02ef84..337c6c72ad 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index 7b9a7c1037..fb1ef447ac 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index cdd4d3ad7e..6309ee7678 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 16fd75ca4f..46729ba377 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 479bb6a715..c52ac7539b 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index dca80f09d6..916bcb0535 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 63c741e7ed..64083cea43 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/patterns/RedshiftColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 6f3f9bb9e2..0cef57df41 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index f830ac6515..5e9fbc6c47 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 79299dacbd..c276795d65 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -323,8 +324,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 6cbaaba999..34d6f5a392 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index f27ba76138..0093a4cc1e 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/pii/RedshiftColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index bdccab9811..776006cff2 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 2b1ac7fdff..dc6aea0e70 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index c6e430bb44..45c91a5d77 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 4aaa390577..9c07c81202 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index e122c38a94..5a56133c3c 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/text/RedshiftColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -281,8 +282,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 9dd9436183..fd2cc882c7 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 278414b0dc..d59871d1ec 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index ecc10bd3d6..417f16b759 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index 417b3e9fe6..28b6c64032 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/uniqueness/RedshiftColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class RedshiftColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 652041f6d8..5bafdfdcc8 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 7e33573f81..74f30a3674 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 738d829183..c9a672f1bc 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index c5b4622d0c..44959043ea 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 2ffe261372..7bbc673e29 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -90,7 +90,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index cbd1faac00..5917fb389f 100644 --- a/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/column/whitespace/RedshiftColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -90,7 +90,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/table/uniqueness/RedshiftTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/table/uniqueness/RedshiftTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..2e0a93830a --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/table/uniqueness/RedshiftTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.redshift.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.redshift.BaseRedshiftIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class RedshiftTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.redshift); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.redshift); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/redshift/sensors/table/uniqueness/RedshiftTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/redshift/sensors/table/uniqueness/RedshiftTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..67a4df83f8 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/redshift/sensors/table/uniqueness/RedshiftTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.redshift.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.redshift.BaseRedshiftIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class RedshiftTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseRedshiftIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.redshift); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.redshift); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 24a5b06666..e15abe2c56 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index c12a6212ae..4f7733eba5 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -298,8 +299,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 73e3222e42..5ce5cb3b16 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -307,8 +308,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 4b34d0310c..fd68ba1e03 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 0446b79866..0f6d40e981 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -349,8 +350,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index 5648276c2a..25eb598069 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 8df304ca14..2b7fd1940e 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/acceptedvalues/SingleStoreDbColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 310f0bfe69..9045938b3b 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index c5264321b4..686a66741d 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/bool/SingleStoreDbColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index d8c344fbd1..db93e8de64 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -226,8 +227,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 1525aeff38..e2c550d6c2 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -264,8 +265,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index b110570b79..109d209e26 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index ad3bf46585..839bec2edc 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/conversions/SingleStoreDbColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index e182211a9c..85bdf799bd 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -142,7 +142,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index dc52c72fc4..33d549b3c6 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -171,7 +172,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -194,7 +195,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -230,7 +231,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -243,8 +244,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index b9fadc2f54..8a375b3f2b 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/customsql/SingleStoreDbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -170,7 +171,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -193,7 +194,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -229,7 +230,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -242,8 +243,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datatype/SingleStoreDbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datatype/SingleStoreDbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index e1a2b0bfd5..463da44965 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datatype/SingleStoreDbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datatype/SingleStoreDbColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -43,7 +43,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,7 +220,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -246,7 +248,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -266,7 +268,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -286,7 +288,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -305,16 +307,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index a227135faf..bcb4b2a1e5 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -286,8 +287,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index f6fe7b468e..467e970913 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 3e33f1a295..c2db3acc9d 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/datetime/SingleStoreDbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -49,6 +49,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -107,7 +108,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -121,7 +122,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,7 +136,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -163,7 +164,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -242,8 +243,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 97a97ff0fc..325bcc5b6e 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -276,8 +277,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 7f306a4c7f..70528a13c4 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/integrity/SingleStoreDbColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -276,8 +277,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index 83ff6846f2..72fd05be69 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -136,12 +137,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -156,14 +157,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -189,7 +190,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -198,15 +199,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 1ab35e30f5..e907ac93eb 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -74,7 +75,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -87,7 +88,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -100,7 +101,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -126,7 +127,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -137,12 +138,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -157,14 +158,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -199,15 +200,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 4b5b80cfcf..7250c26bae 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14.0, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14.0, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14.0, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(14, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 08a2bf37dd..12d7f94b58 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/nulls/SingleStoreDbColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -74,7 +75,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -87,7 +88,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -100,7 +101,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -126,7 +127,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -137,12 +138,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -157,14 +158,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -199,15 +200,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 5cd7e15224..7418441e60 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -249,8 +250,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 724b567397..5dea77c9c7 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -81,7 +82,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 9feb799b31..39a272b13f 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -81,7 +82,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericMeanSensorParametersSpecIntegrationTest.java index d441332901..bc0acad3b3 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 5763ae355c..589eec583f 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 783b321b4b..6ab2ddbc7c 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index 95f9073b3f..955e05d200 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 0ba7f460c3..e642bb84fb 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -238,8 +239,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 1b6f4f4eb3..3a135de739 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 405b55553c..d75197c743 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 89753233c4..1e5434eb91 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 7d8d96e7da..54f7c52cf9 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index e93e6faa05..3173896248 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -249,8 +250,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index c2bc4ea418..975a08ab73 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index d3e28a9feb..36487ebee6 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/numeric/SingleStoreDbColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 92a828a940..298edc3898 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index 1b9d7939c4..57ad801d08 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index f23e970608..43ca5f0aab 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 138ee4b5f7..8a9d07d2ee 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 4580c1a8fe..18c9864685 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 6afb4885b9..1318df730b 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index d6c5baef8d..0f3df7aaee 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,11 +225,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index dcf6dad485..f3b98c3ac3 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,11 +225,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 274601dde3..43e984efda 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index dd05753fdd..cc4b12cf08 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index a877d29f73..644c1cb3b7 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 04d4c427d8..638b963ce9 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 28914482bb..1ba690a40c 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index f80fd875c9..6d83e0c498 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 681c09b279..3e15571532 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/patterns/SingleStoreDbColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 91c90d24c5..9c22343908 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index cb400444bd..0080b2d350 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 638c042dda..2f9c9b6a29 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest class SingleStoreDbColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -326,8 +327,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 66c98ea409..7154976e9e 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 60216fdcd2..e6f2a2abe2 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/pii/SingleStoreDbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -224,8 +225,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index b1d2051ecf..a83c4afd9e 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index a74546e953..096730dd55 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -275,8 +276,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 69effcb6b7..11c0e64298 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -276,8 +277,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index f9119f70a0..7301d5afaf 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -276,8 +277,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 8b2ba37531..e908315202 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/text/SingleStoreDbColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -286,8 +287,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index cd29df5f07..1028a2c949 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 3de10f7826..35f298ba4e 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 2756cc5177..6a199020c3 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index f8e493deef..0b4e5b6be3 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/uniqueness/SingleStoreDbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SingleStoreDbColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index aac15400cb..9a2a4284b4 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 6c797c8dea..1e6dcaa26d 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -226,8 +227,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 6af9260e56..8d2d560ecb 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 5ee5bed099..d2227ffcf7 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -94,7 +95,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -107,7 +108,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -120,7 +121,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -146,7 +147,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -225,8 +226,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index b63468e91d..0d28db63ff 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -86,46 +86,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -138,14 +138,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 5856e2f9ec..3851248a22 100644 --- a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/column/whitespace/SingleStoreDbColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -86,46 +86,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -138,14 +138,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/table/uniqueness/SingleStoreDbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/table/uniqueness/SingleStoreDbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..e62f2089bd --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/table/uniqueness/SingleStoreDbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.singlestore.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.singlestore.BaseSingleStoreDbIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SingleStoreDbTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.redshift); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.redshift); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/singlestore/sensors/table/uniqueness/SingleStoreDbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/table/uniqueness/SingleStoreDbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..361b232185 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/singlestore/sensors/table/uniqueness/SingleStoreDbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.singlestore.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.singlestore.BaseSingleStoreDbIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SingleStoreDbTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseSingleStoreDbIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.redshift); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.redshift); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 8ad7b43c75..07a8ee2db3 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index 604dd2c1d0..ddf44d198f 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -295,8 +296,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 13eb44395c..8f3b43a10c 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 32c96f6d90..654fda9d73 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -300,8 +301,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 191b16652f..ed2f2a520b 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -326,8 +327,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index c7c357be66..6561f925a4 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index baade9dda9..0d8e093e6b 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/acceptedvalues/SnowflakeColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 2ec551f63e..0bd4f60769 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 236f82fed5..c5066ee1f4 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/bool/SnowflakeColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index fd7c1849ab..ab7abdba22 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 794069b98c..96ca2a7294 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index 9a243060b2..4d29a5e218 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index a933439def..f5b7e6ce33 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/conversions/SnowflakeColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index a93dd25e4e..f507eef1ae 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index 04dda18cc3..074b4a460a 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index 046f54b19c..c84dbc3b90 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/customsql/SnowflakeColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datatype/SnowflakeColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datatype/SnowflakeColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 7ef4505314..5034598655 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datatype/SnowflakeColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datatype/SnowflakeColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index 5fe8aefcf0..47270781bb 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index ed132c5fe6..639ee1e3c3 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 80e646c1a3..a801696cc0 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/datetime/SnowflakeColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 4e2155ad74..36ee2cee51 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -271,8 +272,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 5b14339621..9e0e0f0eae 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/integrity/SnowflakeColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -270,8 +271,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index 7d363a3408..01939d18d6 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index b42126753a..bf6ee23fb9 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0F, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0F, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0F, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0F, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 13b5a9b05e..94f0464676 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 0805349a0c..3ac35e1c99 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/nulls/SnowflakeColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 13cc8b651d..e6c9a5b225 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 593db9201d..ff426df363 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 575dc54987..b2286f6ec2 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericMeanSensorParametersSpecIntegrationTest.java index 68f9baa9c2..a572259e65 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index f15716964d..358bd53fad 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index c8fe29ada4..f8a4cf0b5d 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index 33a5795ff0..1a43b68551 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 4f8f408fec..bbadad7041 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index d071e7206c..d65620739d 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 78baccc62f..614a8750d7 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index d2cc62b682..d64202e5dc 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 737f44857b..ba9e49136e 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index 047268b27a..427d44f87a 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericPercentileSensorParametersSpecIntegrationTest.java index 99ae934988..03bed48207 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 557387f6ab..c6cb159c94 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 361b0e030f..65ce504e99 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/numeric/SnowflakeColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 1ffc22e785..d617802e23 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index aa8d2153b4..94197c1afc 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 3653b2463b..c07ea04f4b 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index f92c33f56e..d1cf9af5c3 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index a44102fdda..4ce6832c67 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 10800c5fea..7f1814f0d1 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index c419247270..95f42b33c9 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 97a4e870e9..145863ae58 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 006cff2732..96f80124bf 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index 63bbcefca5..37f1531925 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 2d8cdd645a..0d73fd759b 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 1d74fb8277..29bff377c2 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 1d920ff5ec..fadd772535 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index be9c3351bc..a216ba843f 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 1a83287114..3b58876f07 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/patterns/SnowflakeColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 5d7b25808e..92ca3fa654 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 264ec71023..55ca7eccb3 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 1e45fcf630..189097ed72 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -324,8 +325,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index ec7ddc4b83..fe4f8be6fc 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index ded97f32e6..c7877f7630 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/pii/SnowflakeColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index 2e11e31d2f..f0449bc963 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 5646773e9f..82f4593bc8 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 26ddb955b4..1f45189145 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 892fc6f569..4ba308a442 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 325b186bb5..9dcc8eb802 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/text/SnowflakeColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 20a60efa0d..f01e5a571a 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index bc12cf32d3..fa1f33f94e 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 3a24c76db9..d55b117957 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index 5f0860062e..6d5b978e44 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/uniqueness/SnowflakeColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SnowflakeColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 5219db141a..02e044090b 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 0a7684cb08..4f612af92e 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 2f9c57488c..72b8d2ff3e 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 4c4790251e..de07aaa995 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 38930fae6c..1c69d6bc65 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,14 +83,14 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "email_ok", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 03f51eafaa..0dffad0cdc 100644 --- a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/column/whitespace/SnowflakeColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,14 +83,14 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "email_ok", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0f, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/table/uniqueness/SnowflakeTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/table/uniqueness/SnowflakeTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..ab3e5f42eb --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/table/uniqueness/SnowflakeTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.snowflake.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.snowflake.BaseSnowflakeIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SnowflakeTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.snowflake); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.snowflake); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/snowflake/sensors/table/uniqueness/SnowflakeTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/table/uniqueness/SnowflakeTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..ff01da7d3b --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/snowflake/sensors/table/uniqueness/SnowflakeTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.snowflake.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.snowflake.BaseSnowflakeIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SnowflakeTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseSnowflakeIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.snowflake); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.snowflake); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 0b0c1d1c72..0b15740cc5 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index b17482854b..180e569fbd 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -296,8 +297,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 958f6cc020..fad5733ba6 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 43d58389af..e4f32e0d86 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -306,8 +307,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 9312a162af..c8542c454c 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -326,8 +327,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index a60059ee95..72082e5830 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 3c7e35196e..d7292a8027 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/acceptedvalues/SparkColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 7d820d7fbc..6575cb5176 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 53d22a89fe..820d6b2e57 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/bool/SparkColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 2927e6364e..43e2112f3c 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 688161a362..df7fdcc65a 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index 52dc309cbe..7c7e75d3aa 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index 10c944080c..c7697260ee 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/conversions/SparkColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 551beb4763..9c93931c6f 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index e1eab9a27a..885a662493 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index d5007b95dd..adba723574 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/customsql/SparkColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datatype/SparkColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datatype/SparkColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 3faeb92b45..090de3b3d4 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datatype/SparkColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datatype/SparkColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index 2455f8eaa4..0728b6bcfa 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index 577459f7e0..5049d1d53b 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 1771870b41..1975c86036 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/datetime/SparkColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 7fcd1c0038..0025dc4058 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 6777ea46ba..de1d0687ac 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/integrity/SparkColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index d066911307..73160ef3c1 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index e804469178..c711329edb 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -40,9 +40,10 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index fca46e9906..e1c08a0f95 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 12f9b24ce3..863f408db1 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/nulls/SparkColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -40,9 +40,10 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNullsNullsPercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 2372f037e5..5a43fec2b7 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 4f8ac486fc..a9f150ffa6 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 5bf78c8cc3..c66b237426 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericMeanSensorParametersSpecIntegrationTest.java index aebec8189f..7d8ea0fdd2 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 98f95150e0..3f652adbfc 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 8791097ac1..8d00f35b32 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index 4128ed59af..22209e2e93 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index ebc6977aab..cebabd0dfa 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index c3925d676b..c63db116ae 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 2c7188f7c6..c1b8a3dff9 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index f28b883d48..1649cc1d28 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index 9606c6e9b7..c02abac0c8 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index cee8f803e8..f2d677b0f3 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericPercentileSensorParametersSpecIntegrationTest.java index a600df396f..d1b5517ea2 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 0063749845..5e79decfa5 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 11370ed264..bf0f9e3517 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/numeric/SparkColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index f238a1a1f0..5b42898910 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index a6d6e880c6..71eb230e97 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 34b3a30255..3554e38ee1 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 91b51abc90..277368d92b 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 4c6ff1745f..f37b69c77e 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 35d82e732a..78bde40eb4 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index 15ca4d6380..65af75538e 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index a2d9756ba9..bed00acf0e 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index ba75dc2055..16732c298d 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index f4becbe1d0..1f9e74fc66 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 5fc9897e3d..b2aa41096b 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index d6aaa31523..5f34d228a4 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index c21e11a886..f64c56df90 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index 666d999c87..5beb4bfc77 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 03260cbd4a..c2b7afaab4 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/patterns/SparkColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index af6fbbb374..f90280340d 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index ab70914c2c..0784ce38f4 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index 2273493e73..3c5c628393 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -324,8 +325,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index d7df3f296f..00266d682f 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index d45f4f9e51..a0c9c04cae 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/pii/SparkColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index d7f92ccbe6..014dde3071 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index 3656381a6c..4d9ec2f67c 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 2ae609f834..fb96c4aa71 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 3a91291913..c4ba71d72a 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 33009d6f48..a673d80dfe 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/text/SparkColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 814d1a24e9..2f3c9427a8 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 80520789c6..2d2d425c60 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 7ce6b8084b..50317adc45 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index 0cbfb951f8..d0b203c03f 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/uniqueness/SparkColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SparkColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 624dfdb4a7..890accabe6 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 4b5b4714b3..4473bebae3 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 1805624deb..26b7f3f80f 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index afac417f52..1582a709b3 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index c3633db846..541bec7e84 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,14 +83,14 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "email_ok", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index cab5447d97..9d43893fab 100644 --- a/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/column/whitespace/SparkColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,14 +83,14 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "email_ok", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0d, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/table/uniqueness/SparkTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/table/uniqueness/SparkTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..6224d672db --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/table/uniqueness/SparkTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.spark.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.spark.BaseSparkIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SparkTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.spark); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.spark); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/spark/sensors/table/uniqueness/SparkTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/spark/sensors/table/uniqueness/SparkTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..07299c112f --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/spark/sensors/table/uniqueness/SparkTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.spark.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.spark.BaseSparkIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SparkTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseSparkIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.spark); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.spark); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index b2c4272c29..04beec1ae9 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index d24f81ba20..6893be1929 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -295,8 +296,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 2cdb1ddf5c..1f43ced702 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index 23721507e0..4fcd02efb7 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -281,8 +282,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index 44810a7856..57282e4101 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -362,8 +363,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index e4093e1a86..7b7694b348 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 20c9517a4c..b032737b85 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/acceptedvalues/SqlServerColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index 465891303f..1d04203708 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 6ee383a885..58365c7967 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/bool/SqlServerColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 2537f4cb61..b5ec598369 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 5a02fd9f86..5c944da47f 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index 0cc3a6ceb4..6d0fc89bc6 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index 2c9a97820a..b52cbb18cd 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/conversions/SqlServerColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 8b5fee3bf1..70ab1111f5 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index c0ff001760..d2a65e7103 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index affeb7db8e..ac8c3b4ba9 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/customsql/SqlServerColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datatype/SqlserverColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datatype/SqlserverColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index a54294ff52..d492732240 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datatype/SqlserverColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datatype/SqlserverColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -217,7 +219,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -245,7 +247,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -265,7 +267,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -285,7 +287,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -304,16 +306,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index 41777e5536..42258bba79 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index a508002f0a..97c2b78499 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index dc30a8c121..34081d7cc3 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/datetime/SqlServerColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index 6f1719a2fe..9603d1a789 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 8bf951cf85..7518f8b908 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/integrity/SqlServerColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index 713d35d96f..8bacd0eb76 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 39d8736314..4688af857a 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 9be031921f..ff9308f3d5 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, resultTable.column(0).get(0)); + Assertions.assertEquals(14, resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, resultTable.column(0).get(0)); + Assertions.assertEquals(14, resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, resultTable.column(0).get(0)); + Assertions.assertEquals(14, resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13, resultTable.column(0).get(0)); + Assertions.assertEquals(14, resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index 0d42bc55c7..175bf9b717 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/nulls/SqlServerColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -72,7 +73,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -85,7 +86,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -98,7 +99,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -124,7 +125,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -135,12 +136,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -155,14 +156,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -188,7 +189,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -197,15 +198,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index 0b1340c7e5..ddd872f412 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index 4aaa8d1d01..855860a444 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index be3ab0cbe8..37558c583c 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericMeanSensorParametersSpecIntegrationTest.java index e153c8ab8f..fc065aba9e 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 37f6a53d8b..cf87b25ce8 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 1bf7860177..40f7eb023d 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index e6df6a19e8..12ac8efd32 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index f74cb77225..2100f51093 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 1809b802ac..bd858d28e5 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index 472decd1b6..903e328ec1 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index d42f0314c0..afab742116 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index fe3790f674..35b059c701 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index dab31a3822..d49055cc7a 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericPercentileSensorParametersSpecIntegrationTest.java index ba620deeef..c2e615adc8 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 37cfe20a59..66a9503c5f 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 08a1740566..90c4135320 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/numeric/SqlServerColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 79eb623f76..7785e708dd 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index b74b68699f..6973784436 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 9a94d47584..81a189c049 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 60b2143389..37b0414e60 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index c8f9958333..d4a31416fd 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 1fa306b710..d0d98b7a62 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index fdf45e563c..77d6157039 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 1a7642b22b..2b0fc5ce71 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index cd74057e25..2f9caba79c 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(12, resultTable.column(0).get(0)); + Assertions.assertEquals(8, resultTable.column(0).get(0)); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(12, resultTable.column(0).get(0)); + Assertions.assertEquals(8, resultTable.column(0).get(0)); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(12, resultTable.column(0).get(0)); + Assertions.assertEquals(8, resultTable.column(0).get(0)); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index 5c66a02b3a..530a36c0b8 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index d7a5f4c94a..a3266650fd 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index b5a74b6537..a8c5688026 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index f12dcd589c..2d08cc7a12 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid.domain@.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 7ab59b3526..d35d480573 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/patterns/SqlServerColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -237,8 +238,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid.domain@.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 6beb8d9e47..6734fec7dd 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 1789a27524..3f55c43058 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index f1829879f5..52ed77dcbd 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest class SqlServerColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -323,8 +324,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 35042aae66..460ff2a518 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 72f18b67c2..52bf9dd133 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/pii/SqlServerColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index baf9fa3573..181d8513f7 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index abf0bbd8d4..a712e61613 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 89bea08c7c..b3ef2799a2 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index 0fc49e9633..52de1ea880 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 5bdf7f873a..a844417220 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/text/SqlServerColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index b21e15ca4b..071d3b458a 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 1519f858a8..59e27e54aa 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index 422afd6b3d..0ebd68d182 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index edddd192c9..f31043274e 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/uniqueness/SqlServerColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class SqlServerColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index 6863cb5061..555bfa2cf4 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index b36b4f3eb9..b087f1e26f 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -223,8 +224,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 5f65a35683..3b98489c96 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 7a63e0c979..16b7f05e9a 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 851054eee8..5cbb8c79b0 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -90,7 +90,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index b262889e03..c934b90b47 100644 --- a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/column/whitespace/SqlServerColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -90,7 +90,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -103,7 +103,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -116,7 +116,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -142,7 +142,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/table/uniqueness/SqlServerTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/table/uniqueness/SqlServerTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..20091e8383 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/table/uniqueness/SqlServerTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.sqlserver.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.sqlserver.BaseSqlServerIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SqlServerTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.sqlserver); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.sqlserver); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/table/uniqueness/SqlServerTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/table/uniqueness/SqlServerTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..3226c0836c --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/sqlserver/sensors/table/uniqueness/SqlServerTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.sqlserver.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.sqlserver.BaseSqlServerIntegrationTest; +import com.dqops.testutils.ValueConverter; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class SqlServerTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseSqlServerIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.sqlserver); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.sqlserver); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java index 195fbc6b7b..85a0616262 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnAcceptedValuesTextFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -301,8 +302,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java index be5dd9e4e5..7f13ded204 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNumericExpectedNumbersInUseCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -296,8 +297,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java index 7965e0ce87..bb2d28900f 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNumericNumberFoundInSetPercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -304,8 +305,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(123456789L)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java index d3d31a03bc..0268bfff6a 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnStringsExpectedTextValuesInUseCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -298,8 +299,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java index ed499be6af..d6ca6dff4f 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnStringsExpectedTextsInTopValuesCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -345,8 +346,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("a111a")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java index f28f798608..39815b6ccd 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("CP")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java index 869020857c..8e437d9d96 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/acceptedvalues/TrinoColumnTextTextValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("denar")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java index f453d83ba9..0ca10cc064 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolFalsePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnBoolFalsePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("true")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java index 657e9cb86c..5f1bd4924e 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/bool/TrinoColumnBoolTruePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnBoolTruePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("false")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java index 67059316bf..8507804378 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToBooleanPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("none")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java index 28130e8b46..27f2ef414c 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToDatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -261,8 +262,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java index 5a90e04791..3b0c67bf59 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToFloatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java index e4dcf38035..042a532fa3 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/conversions/TrinoColumnTextTextParsableToIntegerPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("cc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java index 2412cecd8c..815c00fe12 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlAggregatedExpressionSensorParametersSpecIntegrationTest.java @@ -140,7 +140,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(27, resultTable.rowCount()); + Assertions.assertEquals(28, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java index 382f9d3e0d..4524e33a15 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionFailedCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -168,7 +169,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -191,7 +192,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -227,7 +228,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -240,8 +241,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java index c3c1d338dd..2f29812898 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/customsql/TrinoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnSqlConditionPassedPercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -167,7 +168,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -190,7 +191,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -226,7 +227,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(11, resultTable.rowCount()); + Assertions.assertEquals(12, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datatype/TrinoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datatype/TrinoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java index 5ca9179284..02e92e942f 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datatype/TrinoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datatype/TrinoColumnDatatypeStringDatatypeDetectSensorParametersSpecIntegrationTest.java @@ -42,7 +42,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -215,7 +217,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) @@ -243,7 +245,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(2, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); @@ -263,7 +265,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } @Test @@ -283,7 +285,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(4, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -302,16 +304,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2020-01-31T03:51:22Z")); Assertions.assertTrue(sampleValues.contains("abc-001")); - List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(Objects::toString) .collect(Collectors.toSet())); - Assertions.assertEquals(8, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(3465)); + Assertions.assertEquals(9, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains("3465")); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toList()); - Assertions.assertTrue(rowId1Values.contains(3465)); + Assertions.assertTrue(rowId1Values.contains(3)); } } \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java index ac72a978a7..f7f71e8601 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnDatetimeDateInRangePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -283,8 +284,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-01-07")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(4, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(7)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java index 8236303bc3..e744ed864b 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateMatchFormatPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java index 8909380ffb..24050f3d70 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/datetime/TrinoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest.java @@ -27,13 +27,13 @@ import com.dqops.metadata.groupings.DataGroupingDimensionSpec; import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; -import com.dqops.testutils.ValueConverter; -import com.dqops.trino.BaseTrinoIntegrationTest; import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; import com.dqops.sampledata.SampleCsvFileNames; import com.dqops.sampledata.SampleTableMetadata; import com.dqops.sampledata.SampleTableMetadataObjectMother; import com.dqops.sensors.column.datetime.ColumnDatetimeDateValuesInFuturePercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import com.dqops.trino.BaseTrinoIntegrationTest; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnDatetimeDateValuesInFuturePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -118,7 +119,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -132,7 +133,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -160,7 +161,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(6.666, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(6.451, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -239,8 +240,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2999-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java index b3932d749d..e73637027d 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java index 25bb36e864..5159ad5c16 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/integrity/TrinoColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -273,8 +274,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(21)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java index 15f85b07bd..a1313c168d 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNullsNotNullsCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java index 21dc29a0ce..d862aa48a6 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNullsNotNullsPercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(48.0, resultTable.column(0).get(0)); + Assertions.assertEquals(46.153, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java index 2775802b11..601de2c842 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNullsNullsCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(13L, resultTable.column(0).get(0)); + Assertions.assertEquals(14L, resultTable.column(0).get(0)); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java index abb3c580d0..1627e1ca20 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/nulls/TrinoColumnNullsNullsPercentSensorParametersSpecIntegrationTest.java @@ -40,9 +40,10 @@ import org.springframework.boot.test.context.SpringBootTest; import tech.tablesaw.api.Table; -;import java.util.ArrayList; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNullsNullsPercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -71,7 +72,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -84,7 +85,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -97,7 +98,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -123,7 +124,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(52.0, resultTable.column(0).get(0)); + Assertions.assertEquals(53.846, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -134,12 +135,12 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenR SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(1, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); } @@ -154,14 +155,14 @@ void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_the SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(3, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("row_id_1", resultTable.column(1).name()); Assertions.assertEquals("row_id_2", resultTable.column(2).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) @@ -187,7 +188,7 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); - Assertions.assertEquals(10, resultTable.rowCount()); + Assertions.assertEquals(11, resultTable.rowCount()); Assertions.assertEquals(5, resultTable.columnCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); Assertions.assertEquals("sample_index", resultTable.column(1).name()); @@ -196,15 +197,16 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertEquals("row_id_2", resultTable.column(4).name()); Object[] nullValues = resultTable.column("actual_value").asObjectArray(); - Assertions.assertEquals(10, nullValues.length); + Assertions.assertEquals(11, nullValues.length); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(3, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertTrue(groupingLevel1Values.contains(null)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java index a47f3ec613..98fb39c544 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNumericIntegerInRangePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java index fc32b03eaa..77d56364e4 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLatitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java index 58e784d983..ecd8df2965 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericInvalidLongitudeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -78,7 +79,7 @@ void runSensor_onNullData_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(3L, ValueConverter.toDouble(resultTable.column(0).get(0))); + Assertions.assertEquals(0L, ValueConverter.toDouble(resultTable.column(0).get(0))); } @Test @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericMeanSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericMeanSensorParametersSpecIntegrationTest.java index fbb4ced2c4..a40328c324 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericMeanSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericMeanSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java index 1ae28143f9..19868ca105 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java index 03cc225351..319f452e28 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-4.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java index 45375aaddf..251ee8910e 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java index 7e5ee3a52f..0e20ed5f75 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNonNegativePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(-67.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java index 7c4477702f..8f6d6849a6 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java index dada38dde7..7eac6026c6 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberAboveMaxValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(16.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java index 0a219e25e3..3da64d43a8 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValueCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java index fe5d88c305..0d26196cb9 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberBelowMinValuePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -236,8 +237,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java index 292d61507c..7175497615 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnNumericNumberInRangePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -246,8 +247,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.0)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericPercentileSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericPercentileSensorParametersSpecIntegrationTest.java index 954e225087..75cdabc1b9 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericPercentileSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericPercentileSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -214,8 +215,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(15)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java index 6f40583eaa..a2695e5718 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLatitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(91.18464)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java index 0260433248..3d27a930fc 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/numeric/TrinoColumnNumericValidLongitudePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -218,8 +219,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(254.32892)); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java index 9589c0520d..909ec82be7 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java index a774bb6c57..56568e165b 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidEmailFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java index 01d72ae943..f7882367cb 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp4AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("256.212.62.31")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java index 011a75b093..4faa771944 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidIp6AddressFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("b972:ed68:6911:5212:0884:9395:387g:8ad5")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java index 7330712277..db024d3985 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnPatternsInvalidUsaPhoneCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java index 13c989222e..dd9c7c3487 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnPatternsInvalidUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("111111111111111")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java index e01c8b36be..7f4a80fd5f 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnPatternsInvalidUsaZipcodeCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java index 7fbdd1b059..de6d4e7f31 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnPatternsInvalidUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,11 +222,11 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("215388888")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); - Assertions.assertEquals(2, groupingLevel1Values.size()); - Assertions.assertTrue(groupingLevel1Values.contains(1)); + Assertions.assertEquals(1, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) .stream().map(val -> ValueConverter.toInteger(val)) diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java index 96fa539931..aeb643010d 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java index 651d20f94b..cef3ac334e 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsInvalidUuidFormatPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("wrong UUID")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java index 5fca72d4b7..a4cad62d9a 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternCountSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -231,8 +232,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java index 836aaab1bd..f4639c649a 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingDatePatternPercentSensorParametersSpecIntegrationTest.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -228,8 +229,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("33")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java index 4e7abbdb53..c0676d9ca7 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingNamePatternPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("123Szymczak Leszek")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java index c3e8f08c84..4a57e86e7d 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextNotMatchingRegexCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -235,8 +236,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java index 1887a81158..4a09849460 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/patterns/TrinoColumnPatternsTextsNotMatchingRegexPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -234,8 +235,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("invalid@mail@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java index 9207cfd0af..680457388a 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsEmailPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("dot.is.allowed.in.email@mail.com")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java index 3f6c705e07..908d4cc9ec 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp4PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("150.238.182.105")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java index e4dcdc2387..39e8aeabe9 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsIp6PercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -324,8 +325,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("c219:0b3f:96f6:da15:bcac:856a:dd1a:9e71")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java index 7077e14d20..531202e67b 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnPiiContainsUsaPhonePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("+1(231)4561289")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java index cc44022376..c6b2da1640 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/pii/TrinoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnPiiContainsUsaZipcodePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -221,8 +222,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("21531")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java index 5435516fc8..6c3ebba3d0 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java index ccb519c4bf..8c0cbc9325 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthAboveMaxLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abcde")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java index 761fe0aa7f..2042a0ffef 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java index ae47464258..f45f7f9fd0 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthBelowMinLengthPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -272,8 +273,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("abc")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java index 0295071649..c79b42167d 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/text/TrinoColumnTextTextLengthInRangePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -282,8 +283,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("Sample text")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java index 6134d28763..84b8d10cdb 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnUniquenessDistinctCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java index 1451af7cf6..35a00e2f43 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnUniquenessDistinctPercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -220,8 +221,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java index af7a26803d..738c193305 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnUniquenessDuplicateCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java index cdd79a2954..999c7201ef 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/uniqueness/TrinoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest public class TrinoColumnUniquenessDuplicatePercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { @@ -233,8 +234,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("2022-02-01")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(0)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java index c25b26688d..5dfee4c074 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java index 44fb386772..8438586ad6 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceBlankNullPlaceholderTextPercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains("blank")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(2, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java index 018a9bae00..64f09f95c1 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java index 44c319b4b5..475baea3a9 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceTextSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -43,6 +43,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @SpringBootTest @@ -91,7 +92,7 @@ void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -104,7 +105,7 @@ void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -117,7 +118,7 @@ void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -143,7 +144,7 @@ void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(14.8148, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + Assertions.assertEquals(14.285, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test @@ -222,8 +223,8 @@ void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_t Assertions.assertTrue(sampleValues.contains(" name")); List groupingLevel1Values = new ArrayList<>( - List.of(resultTable.column("grouping_level_1").asObjectArray()) - .stream().map(val -> ValueConverter.toInteger(val)) + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) .collect(Collectors.toSet())); Assertions.assertEquals(1, groupingLevel1Values.size()); Assertions.assertTrue(groupingLevel1Values.contains(1)); diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java index 8abbf917fc..a7356a14bf 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextCountSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0L, resultTable.column(0).get(0)); + Assertions.assertEquals(1L, ValueConverter.toLong(resultTable.column(0).get(0))); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java index 8dbe72e5fb..fd702946a5 100644 --- a/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/column/whitespace/TrinoColumnWhitespaceWhitespaceTextPercentSensorParametersSpecIntegrationTest.java @@ -83,46 +83,46 @@ void runSensor_onNullData_thenReturnsValues() { @Test void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( - sampleTableMetadata, "id", this.checkSpec); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForMonitoringCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.daily,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); @@ -135,14 +135,14 @@ void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { @Test void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( - sampleTableMetadata, "id", this.checkSpec, CheckTimeScale.monthly,"date"); + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); Table resultTable = sensorResult.getResultTable(); Assertions.assertEquals(1, resultTable.rowCount()); Assertions.assertEquals("actual_value", resultTable.column(0).name()); - Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + Assertions.assertEquals(3.571, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); } @Test diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/table/uniqueness/TrinoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/table/uniqueness/TrinoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..0fb6e4e21a --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/table/uniqueness/TrinoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.trino.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordCountSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import com.dqops.trino.BaseTrinoIntegrationTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class TrinoTableUniquenessDuplicateRecordCountSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { + private TableDuplicateRecordCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.trino); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordCountSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.trino); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(3L, ValueConverter.toLong(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/trino/sensors/table/uniqueness/TrinoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/trino/sensors/table/uniqueness/TrinoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..9544c8bb05 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/trino/sensors/table/uniqueness/TrinoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,286 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.trino.sensors.table.uniqueness; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.table.checkspecs.uniqueness.TableDuplicateRecordPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.groupings.DataGroupingConfigurationSpec; +import com.dqops.metadata.groupings.DataGroupingDimensionSource; +import com.dqops.metadata.groupings.DataGroupingDimensionSpec; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.table.uniqueness.TableDuplicateRecordPercentSensorParametersSpec; +import com.dqops.testutils.ValueConverter; +import com.dqops.trino.BaseTrinoIntegrationTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@SpringBootTest +public class TrinoTableUniquenessDuplicateRecordPercentSensorParametersSpecIntegrationTest extends BaseTrinoIntegrationTest { + private TableDuplicateRecordPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private TableDuplicateRecordPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.trino); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.sut = new TableDuplicateRecordPercentSensorParametersSpec(); + this.checkSpec = new TableDuplicateRecordPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_onNullData_thenReturnsValues() { + this.sut.setColumns(List.of("int_nulls", "string_nulls")); + + String csvFileName = SampleCsvFileNames.only_nulls; + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile( + csvFileName, ProviderType.trino); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0))); + } + + @Test + void runSensor_whenSensorExecuted_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForProfilingCheck( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedMonitoringMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForMonitoringCheck( + sampleTableMetadata, this.checkSpec,CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.daily, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForPartitionedCheck( + sampleTableMetadata, this.checkSpec, CheckTimeScale.monthly, "date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(10.0, ValueConverter.toDouble(resultTable.column(0).get(0)), 0.001); + } + + @Test + void runSensor_whenErrorSamplingUsesNoColumnsSet_usesAllColumns() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(0, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + } + + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingAndNoIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(1, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithNoGroupingButWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(3, resultTable.rowCount()); + Assertions.assertEquals(3, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("row_id_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_2", resultTable.column(2).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + + Assertions.assertTrue(sampleValues.contains("abcdefgh, e55e")); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(9)); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } + + @Test + void runSensor_whenErrorSamplingSensorExecutedWithDataGroupingAndWithIdColumns_thenReturnsErrorSamples() { + this.sut.setColumns(List.of("length_string", "strings_with_numbers")); + + DataGroupingConfigurationSpec dataGroupingConfigurationSpec = new DataGroupingConfigurationSpec() {{ + setLevel1(new DataGroupingDimensionSpec() {{ + setSource(DataGroupingDimensionSource.column_value); + setColumn("correct"); + }}); + }}; + sampleTableMetadata.getTableSpec().setDefaultDataGroupingConfiguration(dataGroupingConfigurationSpec); + sampleTableMetadata.getTableSpec().getColumns().getAt(0).setId(true); + sampleTableMetadata.getTableSpec().getColumns().getAt(1).setId(true); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableForErrorSampling( + sampleTableMetadata, this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(2, resultTable.rowCount()); + Assertions.assertEquals(4, resultTable.columnCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals("grouping_level_1", resultTable.column(1).name()); + Assertions.assertEquals("row_id_1", resultTable.column(2).name()); + Assertions.assertEquals("row_id_2", resultTable.column(3).name()); + List sampleValues = List.of(resultTable.column("actual_value").asObjectArray()) + .stream().map(val -> String.valueOf(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(sampleValues.contains("abc, d44d")); + Assertions.assertTrue(sampleValues.contains("abcdef, e55e")); + + + List groupingLevel1Values = new ArrayList<>( + Stream.of(resultTable.column("grouping_level_1").asObjectArray()) + .map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toSet())); + Assertions.assertEquals(2, groupingLevel1Values.size()); + Assertions.assertTrue(groupingLevel1Values.contains(0)); + Assertions.assertTrue(groupingLevel1Values.contains(1)); + + List rowId1Values = List.of(resultTable.column("row_id_1").asObjectArray()) + .stream().map(val -> ValueConverter.toInteger(val)) + .collect(Collectors.toList()); + Assertions.assertTrue(rowId1Values.contains(12)); + Assertions.assertTrue(rowId1Values.contains(13)); + } +} diff --git a/dqops/src/main/frontend/src/Routes.tsx b/dqops/src/main/frontend/src/Routes.tsx index 4ad2accf72..027f1a0c33 100644 --- a/dqops/src/main/frontend/src/Routes.tsx +++ b/dqops/src/main/frontend/src/Routes.tsx @@ -20,6 +20,11 @@ const Routes = () => { path={ROUTES.PATTERNS.GLOBAL_INCIDENTS} component={HomePage} /> + {/* Dashboard Layout pages */} 0; + const [advancedPropertiesOpen, setAdvancedPropertiesOpen] = + useState(isExpanded); + + useEffect(() => { + setAdvancedPropertiesOpen(isExpanded); + }, [isExpanded]); + + return ( +
+ {advancedPropertiesOpen ? ( + setAdvancedPropertiesOpen(!advancedPropertiesOpen)} + > + + handleChange({ advanced_properties: properties }) + } + title="Advanced property name" + sharedCredentials={sharedCredentials} + /> + + ) : ( +
+ setAdvancedPropertiesOpen(!advancedPropertiesOpen)} + /> + Advanced properties +
+ )} +
+ ); +} diff --git a/dqops/src/main/frontend/src/components/Button/index.tsx b/dqops/src/main/frontend/src/components/Button/index.tsx index c1c4531be8..24b8486a91 100644 --- a/dqops/src/main/frontend/src/components/Button/index.tsx +++ b/dqops/src/main/frontend/src/components/Button/index.tsx @@ -78,13 +78,14 @@ const Button = ({ 'rounded-lg focus:outline-none flex items-center justify-center', (!className || className.indexOf('py-') === -1) && 'py-2', (!className || className.indexOf('px-') === -1) && 'px-6', - { 'cursor-not-allowed': disabled }, + { 'cursor-not-allowed bg-gray-100': disabled }, { 'flash-red-border': flashRedBorder }, classes === colorsMap.primary.contained && 'hover:bg-[#028770]', classes === colorsMap.primary.outlined && 'hover:bg-[#DDF2EF]' )} disabled={disabled} data-testid={dataTestId} + style={disabled ? { backgroundColor: '#e1e5e9', color: 'black' } : {}} > {loading ? ( diff --git a/dqops/src/main/frontend/src/components/ColumnsRecordDialog/ColumnsRecordDialog.tsx b/dqops/src/main/frontend/src/components/ColumnsRecordDialog/ColumnsRecordDialog.tsx new file mode 100644 index 0000000000..0dd99ddc50 --- /dev/null +++ b/dqops/src/main/frontend/src/components/ColumnsRecordDialog/ColumnsRecordDialog.tsx @@ -0,0 +1,174 @@ +import { Dialog, Tooltip } from '@material-tailwind/react'; +import React, { useEffect, useState } from 'react'; +import { ColumnListModel } from '../../api'; +import { ColumnApiClient } from '../../services/apiClient'; +import { useDecodedParams } from '../../utils'; +import Button from '../Button'; +import Checkbox from '../Checkbox'; +import SvgIcon from '../SvgIcon'; + +interface IColumnsRecordDialogProps { + className?: string; + label?: string; + value: string[]; + tooltipText?: string; + onChange: (value: string[]) => void; + onSave?: () => void; + disabled?: boolean; +} + +type ColumnWithCheckbox = ColumnListModel & { checked: boolean }; + +const ColumnsRecordDialog = ({ + label, + value, + tooltipText, + onChange, + disabled +}: IColumnsRecordDialogProps) => { + const { + connection, + schema, + table + }: { connection: string; schema: string; table: string } = useDecodedParams(); + const [columnsWithCheckboxes, setColumnsWithCheckboxes] = useState< + ColumnWithCheckbox[] + >([]); + const [open, setOpen] = useState(false); + + const handleSave = () => { + const selectedColumns = columnsWithCheckboxes + .filter((column) => column.checked) + .map((column) => column.column_name ?? ''); + onChange(selectedColumns); + setOpen(false); + }; + + useEffect(() => { + const getColumns = async () => { + await ColumnApiClient.getColumns(connection, schema, table).then( + (res) => { + const columnsWithCheckboxes = res.data + .map((column) => ({ + ...column, + checked: value.includes(column.column_name ?? '') + })) + .sort((a, b) => + (a.column_name ?? '')?.localeCompare(b.column_name ?? '') + ); + setColumnsWithCheckboxes(columnsWithCheckboxes); + } + ); + }; + if (open) getColumns(); + }, [open]); + + return ( +
+
+ {label && ( + + )} +
+
+
+ {value?.join(', ')?.slice(0, 200)} + {value.join(', ').length > 200 && '...'} +
+
+ !disabled && setOpen(true)} + /> +
+
+ setOpen(false)} + style={{ width: '200px !important', minWidth: '25%', maxWidth: '25%' }} + className="pt-6" + > +
+
+ + + + + + + {columnsWithCheckboxes.map((column) => ( + + + + ))} + +
Column
+
+ { + const newColumns = columnsWithCheckboxes.map((c) => + c.column_name === column.column_name + ? { ...c, checked: value } + : c + ); + setColumnsWithCheckboxes(newColumns); + }} + /> +
+ {column.column_name} +
+
+
+ +
+
+
+
+ + + ); +}; + +export default ColumnsRecordDialog; diff --git a/dqops/src/main/frontend/src/components/Connection/CommentsView/CommentItem.tsx b/dqops/src/main/frontend/src/components/Connection/CommentsView/CommentItem.tsx index cbe9718155..8a168c603c 100644 --- a/dqops/src/main/frontend/src/components/Connection/CommentsView/CommentItem.tsx +++ b/dqops/src/main/frontend/src/components/Connection/CommentsView/CommentItem.tsx @@ -48,6 +48,7 @@ const CommentItem = ({
onRemove(idx)} color="teal" diff --git a/dqops/src/main/frontend/src/components/Connection/CommentsView/index.tsx b/dqops/src/main/frontend/src/components/Connection/CommentsView/index.tsx index 3e0ac84ad9..a00b1a8fb8 100644 --- a/dqops/src/main/frontend/src/components/Connection/CommentsView/index.tsx +++ b/dqops/src/main/frontend/src/components/Connection/CommentsView/index.tsx @@ -99,6 +99,7 @@ const CommentsView = ({
+ +
+ ) : ( + <> +
+ +
+
+ ); +}; diff --git a/dqops/src/main/frontend/src/components/Connection/ConnectionView/IncidentsNotificationsView.tsx b/dqops/src/main/frontend/src/components/Connection/ConnectionView/IncidentsNotificationsView.tsx index 52d3e69d2d..b50c469cf7 100644 --- a/dqops/src/main/frontend/src/components/Connection/ConnectionView/IncidentsNotificationsView.tsx +++ b/dqops/src/main/frontend/src/components/Connection/ConnectionView/IncidentsNotificationsView.tsx @@ -1,17 +1,14 @@ import clsx from 'clsx'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect } from 'react'; import { useSelector } from 'react-redux'; import { ConnectionIncidentGroupingSpec, ConnectionIncidentGroupingSpecGroupingLevelEnum, - ConnectionIncidentGroupingSpecMinimumSeverityEnum, - FilteredNotificationModel, - IncidentNotificationSpec + ConnectionIncidentGroupingSpecMinimumSeverityEnum } from '../../../api'; import { useActionDispatch } from '../../../hooks/useActionDispatch'; import { getConnectionIncidentGrouping, - setActiveFirstLevelTab, setUpdateIncidentGroup, updateConnectionIncidentGrouping } from '../../../redux/actions/source.actions'; @@ -20,28 +17,13 @@ import { getFirstLevelActiveTab, getFirstLevelState } from '../../../redux/selectors'; -import { FilteredNotificationsConfigurationsClient } from '../../../services/apiClient'; import { CheckTypes } from '../../../shared/routes'; import { useDecodedParams } from '../../../utils'; -import Button from '../../Button'; import Checkbox from '../../Checkbox'; -import Loader from '../../Loader'; import NumberInput from '../../NumberInput'; import Select from '../../Select'; -import SvgIcon from '../../SvgIcon'; import ConnectionActionGroup from './ConnectionActionGroup'; -import CreateNotificationPattern from './NotificationPattern/CreateNotificationPattern'; -import NotificationPatternTable from './NotificationPattern/NotificationPatternTable'; -type TNotificationPattern = FilteredNotificationModel & { - connection?: string; - schema?: string; - table?: string; - qualityDimension?: string; - checkCategory?: string; - checkName?: string; - checkType?: string; - highestSeverity?: number; -}; + const groupLevelOptions = Object.values( ConnectionIncidentGroupingSpecGroupingLevelEnum ).map((item) => ({ @@ -66,87 +48,10 @@ export const IncidentsNotificationsView = () => { checkTypes }: { connection: string; checkTypes: CheckTypes } = useDecodedParams(); const dispatch = useActionDispatch(); - const { - incidentGrouping, - isUpdatedIncidentGroup, - isUpdating, - incidentFilters - } = useSelector(getFirstLevelState(checkTypes)); + const { incidentGrouping, isUpdatedIncidentGroup, isUpdating } = useSelector( + getFirstLevelState(checkTypes) + ); const firstLevelActiveTab = useSelector(getFirstLevelActiveTab(checkTypes)); - const [patternNameEdit, setPatternNameEdit] = useState(''); - const [loading, setLoading] = useState(false); - - const [ - filteredNotificationsConfigurations, - setFilteredNotificationsConfigurations - ] = useState>([]); - const [addNotificationPattern, setAddNotificationPattern] = useState(false); - const [patternProp, setPatternProp] = useState< - FilteredNotificationModel | undefined - >(); - - useEffect(() => { - if (!incidentFilters) return; - if (incidentFilters.notificationName) { - setPatternNameEdit(incidentFilters.notificationName); - } else { - setPatternProp({ - name: '', - filter: { - connection: incidentFilters.connection, - schema: incidentFilters.schema, - table: incidentFilters.table, - qualityDimension: incidentFilters.qualityDimension, - checkCategory: incidentFilters.checkCategory, - checkName: incidentFilters.check, - checkType: incidentFilters.checkType - }, - priority: 1000 - }); - setAddNotificationPattern(true); - } - }, [incidentFilters]); - - const getConnectionFilteredNotificationsConfigurations = async () => { - setLoading(true); - FilteredNotificationsConfigurationsClient.getConnectionFilteredNotificationsConfigurations( - connection - ) - .then((response) => { - const patterns: TNotificationPattern[] = response.data.map((x) => { - return { - ...x, - connection: x.filter?.connection || '', - schema: x.filter?.schema || '', - table: x.filter?.table || '', - qualityDimension: x.filter?.qualityDimension || '', - checkCategory: x.filter?.checkCategory || '', - checkName: x.filter?.checkName || '', - checkType: x.filter?.checkType || '' - }; - }); - setFilteredNotificationsConfigurations(patterns); - }) - .finally(() => setLoading(false)); - }; - - useEffect(() => { - getConnectionFilteredNotificationsConfigurations(); - }, [connection]); - - const createNotificationPattern = () => { - setAddNotificationPattern(true); - }; - const onBack = () => { - console.log(firstLevelActiveTab); - if (connection) { - dispatch(setActiveFirstLevelTab(checkTypes, firstLevelActiveTab)); - } - setPatternNameEdit(''); - setPatternProp(undefined); - setAddNotificationPattern(false); - getConnectionFilteredNotificationsConfigurations(); - }; useEffect(() => { dispatch( @@ -163,20 +68,6 @@ export const IncidentsNotificationsView = () => { ); }; - const onChangeConnectionDefaultAdresses = ( - obj: Partial - ) => { - dispatch( - setUpdateIncidentGroup(checkTypes, firstLevelActiveTab, { - ...(incidentGrouping || {}), - incident_notification: { - ...(incidentGrouping?.incident_notification || {}), - ...obj - } - }) - ); - }; - const onUpdate = () => { dispatch( updateConnectionIncidentGrouping( @@ -188,14 +79,6 @@ export const IncidentsNotificationsView = () => { ); }; - const defaultConnectionAdressess = incidentGrouping?.incident_notification; - if (loading) { - return ( -
- -
- ); - } return (
{ : '' )} > - {!addNotificationPattern && !patternNameEdit && ( - - )} + -
- {addNotificationPattern || patternNameEdit ? ( -
-
-
- + <> +
+ onChange({ minimum_severity: value })} + className="text-sm" + menuClassName="!top-14" />
- ) : ( - <> -
- onChange({ minimum_severity: value })} - className="text-sm" - menuClassName="!top-14" +
+
+

Maximum incident duration

+
+ + onChange({ max_incident_length_days: value }) + } /> + days. After this time, DQOps creates a new incident.
-
-

Create separate incidents for each data group

-
- - onChange({ divide_by_data_groups: value }) - } - /> -
-
-
-

Maximum incident duration

-
- - onChange({ max_incident_length_days: value }) - } - /> - - days. After this time, DQOps creates a new incident. - -
-
-
-

Mute data quality issues for

-
- {}} - /> - - {' '} - days. If the incident is muted, DQOps will not create a new - one. - -
+
+
+

Mute data quality issues for

+
+ onChange({ mute_for_days: value })} + /> + + {' '} + days. If the incident is muted, DQOps will not create a new one. +
- -
+
); diff --git a/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/DefaultPatternTarget.tsx b/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/DefaultPatternTarget.tsx index 18be896868..17dd59cf45 100644 --- a/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/DefaultPatternTarget.tsx +++ b/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/DefaultPatternTarget.tsx @@ -1,8 +1,10 @@ import React from 'react'; import { FilteredNotificationModel, - NotificationFilterSpec + NotificationFilterSpec, + NotificationFilterSpecCheckTypeEnum } from '../../../../api'; +import { useDecodedParams } from '../../../../utils'; import Checkbox from '../../../Checkbox'; import SectionWrapper from '../../../Dashboard/SectionWrapper'; import Input from '../../../Input'; @@ -22,6 +24,7 @@ export default function DefaultPatternTarget({ onChange, onChangePatternFilter }: TDefaultCheckTargetConfigurationProps) { + const { connection }: { connection: string } = useDecodedParams(); return (
@@ -66,8 +69,7 @@ export default function DefaultPatternTarget({ />
-
- Disabled +
@@ -76,11 +78,9 @@ export default function DefaultPatternTarget({ }) } /> + Disabled
-
- - Process additional notification filters - +
@@ -89,9 +89,11 @@ export default function DefaultPatternTarget({ }) } /> + + Process additional notification filters +
-
- Do not create incidents +
@@ -100,6 +102,7 @@ export default function DefaultPatternTarget({ }) } /> + Do not create incidents
@@ -107,13 +110,17 @@ export default function DefaultPatternTarget({
Connection
- - onChangePatternFilter({ connection: e.target.value }) - } - /> + {connection ? ( + + ) : ( + + onChangePatternFilter({ connection: e.target.value }) + } + /> + )}
@@ -135,14 +142,14 @@ export default function DefaultPatternTarget({
- Table + Schema
- onChangePatternFilter({ table: e.target.value }) + onChangePatternFilter({ schema: e.target.value }) } - className="w-11/12" />
@@ -161,18 +168,17 @@ export default function DefaultPatternTarget({
- Schema + Table
- onChangePatternFilter({ schema: e.target.value }) + onChangePatternFilter({ table: e.target.value }) } + className="w-11/12" />
-
Check category
@@ -204,11 +210,18 @@ export default function DefaultPatternTarget({
Check type
- ({ + label: x, + value: x, + })), + ]} value={pattern?.filter?.checkType} - onChange={(e) => + onChange={(value) => onChangePatternFilter({ - checkType: e.target.value + checkType: value as NotificationFilterSpecCheckTypeEnum }) } /> @@ -232,6 +245,19 @@ export default function DefaultPatternTarget({ highestSeverity: value }) } + /> +
+
+
+ Data group name +
+ + onChangePatternFilter({ + dataGroupName: value.target.value + }) + } className="w-49" />
diff --git a/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/NotificationPatternTable.tsx b/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/NotificationPatternTable.tsx index 10d2d1c911..665ad7c365 100644 --- a/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/NotificationPatternTable.tsx +++ b/dqops/src/main/frontend/src/components/Connection/ConnectionView/NotificationPattern/NotificationPatternTable.tsx @@ -1,13 +1,15 @@ +import { IconButton } from '@material-tailwind/react'; import clsx from 'clsx'; -import React, { useState } from 'react'; +import React, { useEffect, useMemo, useState } from 'react'; import { FilteredNotificationModel } from '../../../../api'; import { FilteredNotificationsConfigurationsClient } from '../../../../services/apiClient'; import { getSeverity, sortPatterns } from '../../../../utils'; -import Button from '../../../Button'; import ConfirmDialog from '../../../CustomTree/ConfirmDialog'; +import Loader from '../../../Loader'; +import ClientSidePagination from '../../../Pagination/ClientSidePagination'; import SvgIcon from '../../../SvgIcon'; -const headerElements = [ +const HEADER_ELEMENTS = [ { label: 'Notification filters', key: 'name' }, { label: 'Priority', key: 'priority' }, { label: 'Connection', key: 'connection' }, @@ -17,7 +19,8 @@ const headerElements = [ { label: 'Check category', key: 'checkCategory' }, { label: 'Check name', key: 'checkName' }, { label: 'Check type', key: 'checkType' }, - { label: 'Highest severity', key: 'highestSeverity' } + { label: 'Highest severity', key: 'highestSeverity' }, + { label: 'Action', key: 'action' } ]; type TNotificationPattern = FilteredNotificationModel & { @@ -36,153 +39,245 @@ export default function NotificationPatternTable({ filteredNotificationsConfigurations, onChange, setPatternNameEdit, - connection + connection, + loading }: { filteredNotificationsConfigurations: Array; onChange: (data: any) => void; setPatternNameEdit: (patternName: string) => void; connection?: string; + loading?: boolean; }) { - const [dir, setDir] = useState<'asc' | 'desc'>('desc'); + const [dir, setDir] = useState<'asc' | 'desc'>('asc'); const [notificationPatternDelete, setPatternDelete] = useState(''); const [indexSortingElement, setIndexSortingElement] = useState(1); + const [headerItems, setHeaderItems] = useState(HEADER_ELEMENTS); + const [displayedPatterns, setDisplayedPatterns] = useState([]); // State for displayed patterns - const sortPreparedPattern = ( - elem: any, - index: number, - dir: 'asc' | 'desc' - ) => { + const getFilteredItemsBasedOnWidth = () => { + const width = window.innerWidth; + + return HEADER_ELEMENTS.filter((item) => { + if ( + width < 1800 && + [ + 'qualityDimension', + 'checkCategory', + 'checkName', + 'checkType', + 'highestSeverity' + ].includes(item.key) + ) + return false; + if ( + width < 1900 && + ['checkCategory', 'checkName', 'checkType', 'highestSeverity'].includes( + item.key + ) + ) + return false; + if (width < 2000 && ['checkType', 'highestSeverity'].includes(item.key)) + return false; + return true; + }); + }; + + const updateHeaderItems = () => { + setHeaderItems(getFilteredItemsBasedOnWidth()); + }; + + useEffect(() => { + updateHeaderItems(); + window.addEventListener('resize', updateHeaderItems); + return () => window.removeEventListener('resize', updateHeaderItems); + }, []); + + const handleSort = (elem: { label: string; key: string }, index: number) => { + const newDir = dir === 'asc' ? 'desc' : 'asc'; onChange( sortPatterns( filteredNotificationsConfigurations, elem.key as keyof TNotificationPattern, - dir + newDir ) - ), - setDir(dir === 'asc' ? 'desc' : 'asc'), - setIndexSortingElement(index); + ); + setDir(newDir); + setIndexSortingElement(index); }; - const deletePattern = (patternName: string) => { - if (connection) { - FilteredNotificationsConfigurationsClient.deleteConnectionFilteredNotificationConfiguration( - connection, - patternName - ).then(() => { - onChange( - filteredNotificationsConfigurations.filter( - (pattern) => pattern.name !== patternName - ) + const handleDeletePattern = (patternName: string) => { + const deleteAction = connection + ? FilteredNotificationsConfigurationsClient.deleteConnectionFilteredNotificationConfiguration( + connection, + patternName + ) + : FilteredNotificationsConfigurationsClient.deleteDefaultFilteredNotificationConfiguration( + patternName ); - }); - } else { - FilteredNotificationsConfigurationsClient.deleteDefaultFilteredNotificationConfiguration( - patternName - ).then(() => { - onChange( - filteredNotificationsConfigurations.filter( - (pattern) => pattern.name !== patternName - ) - ); - }); - } + + deleteAction.then(() => { + onChange( + filteredNotificationsConfigurations.filter( + (pattern) => pattern.name !== patternName + ) + ); + }); }; + const sortedNotifications = useMemo( + () => [...filteredNotificationsConfigurations, { name: 'default' }], + [filteredNotificationsConfigurations] + ); + if (loading) { + return ( +
+ +
+ ); + } return ( - - - - {headerElements.map((elem, index) => ( -
-
-
{elem.label}
-
- {!(indexSortingElement === index && dir === 'asc') ? ( - sortPreparedPattern(elem, index, 'desc')} - /> - ) : ( -
+ <> + + + + {headerItems.map((elem, index) => ( + - ))} - - - - {[ - ...filteredNotificationsConfigurations, - { - name: 'default' - } - ].map((notificationPattern, index) => ( - - - - - - - - - - - - - {notificationPattern.name !== 'default' && ( - + +
+ +
+ {displayedPatterns.map((notificationPattern, index) => ( + + {getFilteredItemsBasedOnWidth().map((elem) => ( + - )} - - ))} - 0} - onConfirm={async () => { - deletePattern(notificationPatternDelete ?? ''), + > + {elem.key === 'name' && notificationPattern.name} + {elem.key === 'priority' && ( +
+ {notificationPattern.priority}{' '} +
+ )} + {elem.key === 'connection' && notificationPattern.connection} + {elem.key === 'schema' && notificationPattern.schema} + {elem.key === 'table' && notificationPattern.table} + {elem.key === 'qualityDimension' && + notificationPattern.qualityDimension} + {elem.key === 'checkCategory' && + notificationPattern.checkCategory} + {elem.key === 'checkName' && notificationPattern.checkName} + {elem.key === 'checkType' && notificationPattern.checkType} + {elem.key === 'highestSeverity' && + getSeverity( + String(notificationPattern.highestSeverity ?? '') + )} + {elem.key === 'action' && ( +
+ + setPatternNameEdit(notificationPattern.name ?? '') + } + size="sm" + color="teal" + className={clsx( + '!shadow-none hover:!shadow-none hover:bg-[#028770]', + filteredNotificationsConfigurations.length === 0 && + 'ml-1.5' + )} + > + + + {notificationPattern.name !== 'default' && ( + + setPatternDelete(notificationPattern.name ?? '') + } + size="sm" + color="teal" + className="!shadow-none hover:!shadow-none hover:bg-[#028770]" + > + + + )} +
+ )} + + ))} + + ))} + + 0} + onConfirm={async () => { + handleDeletePattern(notificationPatternDelete); setPatternDelete(''); - }} - onClose={() => setPatternDelete('')} - message={`Are you sure you want to delete the ${notificationPatternDelete} notification filter?`} + }} + onClose={() => setPatternDelete('')} + message={`Are you sure you want to delete the ${notificationPatternDelete} notification filter?`} + /> + +
+
sortPreparedPattern(elem, index, 'asc')} - /> - ) : ( -
+ > +
0 && + 'ml-4', + elem.key === 'priority' && + 'text-right flex items-center justify-end !max-w-10' + )} + > + {elem.label} +
+ {elem.key !== 'action' && ( +
+ {!(indexSortingElement === index && dir === 'asc') ? ( + handleSort(elem, index)} + /> + ) : ( +
+ )} + {!(indexSortingElement === index && dir === 'desc') ? ( + handleSort(elem, index)} + /> + ) : ( +
+ )} +
)}
-
-
setPatternNameEdit(notificationPattern.name ?? '')} - > - {notificationPattern.name} - {notificationPattern.priority}{notificationPattern?.connection}{notificationPattern?.schema}{notificationPattern?.table}{notificationPattern?.qualityDimension}{notificationPattern?.checkCategory}{notificationPattern?.checkName}{notificationPattern?.checkType} - {getSeverity(String(notificationPattern?.highestSeverity ?? ''))} - - -
- setPatternDelete(notificationPattern.name ?? '') + elem.key === 'name' && + setPatternNameEdit(notificationPattern.name ?? '') } - /> -
+
+ - -
+
+ ); } diff --git a/dqops/src/main/frontend/src/components/Connection/ConnectionView/SchemasView.tsx b/dqops/src/main/frontend/src/components/Connection/ConnectionView/SchemasView.tsx index f6b2c8ac86..73c5505db3 100644 --- a/dqops/src/main/frontend/src/components/Connection/ConnectionView/SchemasView.tsx +++ b/dqops/src/main/frontend/src/components/Connection/ConnectionView/SchemasView.tsx @@ -80,7 +80,7 @@ const SchemasView = () => {
{isSourceScreen && } - + {itemsToRender.map((itemData, jIndex) => ( diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableComparisonUtils.ts b/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableComparisonUtils.ts index b7140422ab..fcb3b1926c 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableComparisonUtils.ts +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableComparisonUtils.ts @@ -64,13 +64,13 @@ export const calculateColor = ( } if (colorVar?.fatals && Number(colorVar.fatals) !== 0) { - return 'bg-red-200'; + return 'bg-[#EF8079]'; } else if (colorVar?.errors && Number(colorVar.errors) !== 0) { - return 'bg-orange-200'; + return 'bg-[#EFB87E]'; } else if (colorVar?.warnings && Number(colorVar.warnings) !== 0) { - return 'bg-yellow-200'; + return 'bg-[#EFEC82]'; } else if (colorVar?.valid_results && Number(colorVar.valid_results) !== 0) { - return 'bg-green-200'; + return 'bg-[#5CBCAB]'; } else { return ''; } @@ -84,7 +84,8 @@ export const onUpdate = ( timePartitioned: 'daily' | 'monthly' | undefined, reference: TableComparisonModel | undefined, handleChange: (value: CheckContainerModel) => Promise, - tableChecksToUpdate: any + tableChecksToUpdate: any, + callback?: () => Promise ) => { if (checkTypes === CheckTypes.PROFILING) { TableComparisonsApi.updateTableComparisonProfiling( @@ -93,9 +94,13 @@ export const onUpdate = ( table, reference?.table_comparison_configuration_name ?? '', reference - ).catch((err) => { - console.error(err); - }); + ) + .then(() => { + callback && callback(); + }) + .catch((err) => { + console.error(err); + }); } else if (checkTypes === CheckTypes.MONITORING) { if (timePartitioned === 'daily') { TableComparisonsApi.updateTableComparisonMonitoringDaily( @@ -104,9 +109,13 @@ export const onUpdate = ( table, reference?.table_comparison_configuration_name ?? '', reference - ).catch((err) => { - console.error(err); - }); + ) + .then(() => { + callback && callback(); + }) + .catch((err) => { + console.error(err); + }); } else if (timePartitioned === 'monthly') { TableComparisonsApi.updateTableComparisonMonitoringMonthly( connection, @@ -114,9 +123,13 @@ export const onUpdate = ( table, reference?.table_comparison_configuration_name ?? '', reference - ).catch((err) => { - console.error(err); - }); + ) + .then(() => { + callback && callback(); + }) + .catch((err) => { + console.error(err); + }); } } else if (checkTypes === CheckTypes.PARTITIONED) { if (timePartitioned === 'daily') { diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableLevelResults.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableLevelResults.tsx index 20e653d688..8cf979047e 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableLevelResults.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableLevelResults.tsx @@ -10,35 +10,35 @@ export default function TableLevelResults({ type }: TTableLevelResults) { return ( -
+
+ - + { tableComparisonResults?.table_comparison_results?.[type ?? ''] ?.valid_results } - - {tableComparisonResults?.table_comparison_results?.[type ?? '']?.errors} - - - {tableComparisonResults?.table_comparison_results?.[type ?? '']?.fatals} - - { tableComparisonResults?.table_comparison_results?.[type ?? ''] ?.warnings } + + {tableComparisonResults?.table_comparison_results?.[type ?? '']?.errors} + + + {tableComparisonResults?.table_comparison_results?.[type ?? '']?.fatals} + {type.includes('row') ? (
Schema name @@ -97,11 +97,13 @@ const SchemasView = () => { label={item.schema_name} variant="text" className="underline px-2 text-sm" - onClick={() => goToTable(item.schema_name ?? '', 'tables')} + onClick={() => + goToTable(item.schema_name ?? '', 'data-quality-summary') + } /> {isSourceScreen ? ( - + - Results:Results - Valid: + Correct results: - Errors: - Fatal: + Warning: + Errors: + Fatal errors: - + Show mismatches
; showRowCount: boolean; onUpdateChecksUI: ( - checksUI: any, type: 'row' | 'column', disabled?: boolean, severity?: TSeverityValues ) => void; - checksUI: any; setIsUpdated: React.Dispatch>; tableComparisonResults: any; showColumnCount: boolean; @@ -33,7 +31,6 @@ const TableRow: React.FC = ({ settableLevelComparisonExtended, showRowCount, onUpdateChecksUI, - checksUI, setIsUpdated, tableComparisonResults, showColumnCount, @@ -88,7 +85,7 @@ const TableRow: React.FC = ({ { - onUpdateChecksUI(checksUI, 'row', checked); + onUpdateChecksUI('row', checked); setIsUpdated(true); }} /> @@ -121,7 +118,7 @@ const TableRow: React.FC = ({ { - onUpdateChecksUI(checksUI, 'column', checked); + onUpdateChecksUI('column', checked); setIsUpdated(true); }} /> diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableReferenceComparisons.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableReferenceComparisons.tsx index 3a9c3ef417..be164d8b24 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableReferenceComparisons.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableComparison/TableReferenceComparisons.tsx @@ -132,71 +132,29 @@ export const TableReferenceComparisons = ({ setIsEditing(true); setIsCreting(false); } else { - let url = ''; - if (checkTypes === CheckTypes.PROFILING) { - url = `${ROUTES.TABLE_LEVEL_PAGE( - checkTypes, - connection, - schema, - table, - 'table-comparisons' - )}`; - dispatch( - addFirstLevelTab(checkTypes, { - url, - value: ROUTES.TABLE_LEVEL_VALUE( - checkTypes, - connection, - schema, - table - ), - state: {}, - label: table - }) - ); - } else if (timePartitioned === 'daily') { - url = `${ROUTES.TABLE_LEVEL_PAGE( - checkTypes, - connection, - schema, - table, - 'daily_comparisons' - )}`; - dispatch( - addFirstLevelTab(checkTypes, { - url, - value: ROUTES.TABLE_LEVEL_VALUE( - checkTypes, - connection, - schema, - table - ), - state: {}, - label: table - }) - ); - } else if (timePartitioned === 'monthly') { - url = `${ROUTES.TABLE_LEVEL_PAGE( - checkTypes, - connection, - schema, - table, - 'monthly_comparisons' - )}`; - dispatch( - addFirstLevelTab(checkTypes, { - url, - value: ROUTES.TABLE_LEVEL_VALUE( - checkTypes, - connection, - schema, - table - ), - state: {}, - label: table - }) - ); - } + const url = `${ROUTES.TABLE_LEVEL_PAGE( + checkTypes, + connection, + schema, + table, + 'table-comparisons' + )}`; + dispatch( + addFirstLevelTab(checkTypes, { + url, + value: ROUTES.TABLE_LEVEL_VALUE( + checkTypes, + connection, + schema, + table + ), + state: { + checksUI + }, + label: table + }) + ); + if (isCreating === true) { getNewTableComparison(); } diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableDetails.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableDetails.tsx index bbc65aa81a..924f3faa0f 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableDetails.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableDetails.tsx @@ -6,6 +6,7 @@ import { ConnectionSpecProviderTypeEnum, DuckdbParametersSpecFilesFormatTypeEnum, FileFormatSpec, + SharedCredentialListModel, TableListModelProfilingChecksResultTruncationEnum } from '../../../api'; import { TConfiguration } from '../../../components/FileFormatConfiguration/TConfiguration'; @@ -20,9 +21,13 @@ import { getFirstLevelActiveTab, getFirstLevelState } from '../../../redux/selectors'; -import { ConnectionApiClient } from '../../../services/apiClient'; +import { + ConnectionApiClient, + SharedCredentialsApi +} from '../../../services/apiClient'; import { CheckTypes } from '../../../shared/routes'; import { useDecodedParams } from '../../../utils'; +import AdvancedProperties from '../../AdvancedProperties/AdvancedProperties'; import Checkbox from '../../Checkbox'; import FileFormatConfiguration from '../../FileFormatConfiguration/FileFormatConfiguration'; import FilePath from '../../FileFormatConfiguration/FilePath'; @@ -46,16 +51,20 @@ const TableDetails = () => { const { tableBasic, isUpdating, isUpdatedTableBasic } = useSelector( getFirstLevelState(checkTypes) ); - const format = - (Object.keys(tableBasic?.file_format ?? {}).find((x) => - x.includes('format') - ) as DuckdbParametersSpecFilesFormatTypeEnum) ?? - DuckdbParametersSpecFilesFormatTypeEnum.csv; - + const format = Object.keys(tableBasic?.file_format ?? {}).find((x) => { + return DuckdbParametersSpecFilesFormatTypeEnum[ + x as keyof typeof DuckdbParametersSpecFilesFormatTypeEnum + ]; + }); const [connectionModel, setConnectionModel] = useState({}); - + const [sharedCredentials, setSharedCredentials] = useState< + SharedCredentialListModel[] + >([]); const [fileFormatType, setFileFormatType] = - useState(format); + useState( + (format as DuckdbParametersSpecFilesFormatTypeEnum) ?? + DuckdbParametersSpecFilesFormatTypeEnum.csv + ); const onChangeConfiguration = (params: Partial) => { // setConfiguration((prev) => ({ @@ -89,9 +98,26 @@ const TableDetails = () => { setConnectionModel(res.data) ); }; + const getSharedCredentials = async () => { + await SharedCredentialsApi.getAllSharedCredentials().then((res) => + setSharedCredentials(res.data) + ); + }; + getSharedCredentials(); getConnectionBasic(); }, [checkTypes, connection, schema, table]); + useEffect(() => { + if (!tableBasic || !tableBasic?.file_format) { + return; + } + + setFileFormatType( + (format as DuckdbParametersSpecFilesFormatTypeEnum) ?? + DuckdbParametersSpecFilesFormatTypeEnum.csv + ); + }, [tableBasic?.file_format]); + const handleChange = (obj: any) => { dispatch( setUpdatedTableBasic(checkTypes, firstLevelActiveTab, { @@ -192,6 +218,11 @@ const TableDetails = () => { > {TableDetailBody({ tableBasic, handleChange })}
+ {connectionModel.provider_type === ConnectionSpecProviderTypeEnum.duckdb && ( - - Do not collect error samples for profiling checks + + + Do not collect error samples for profiling checks +
handleChange({ do_not_collect_error_samples_in_profiling: value })} + onChange={(value) => + handleChange({ + do_not_collect_error_samples_in_profiling: value + }) + } checked={tableBasic?.do_not_collect_error_samples_in_profiling} />
- Always collect error samples for scheduled monitoring checks + + Always collect error samples for scheduled monitoring checks +
handleChange({ always_collect_error_samples: value })} - checked={tableBasic?.always_collect_error_samples} + onChange={(value) => + handleChange({ always_collect_error_samples: value }) + } + checked={tableBasic?.always_collect_error_samples} />
diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/CurrentTableStatus.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/CurrentTableStatus.tsx index d8a91629e7..6b80fd5140 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/CurrentTableStatus.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/CurrentTableStatus.tsx @@ -11,11 +11,11 @@ export default function CurrentTableStatus({ return (
-
Status:
+
Status
{tableDataQualityStatus.current_severity}
-
Last check executed at:
+
Last check executed at
{moment(tableDataQualityStatus.last_check_executed_at).format( 'YYYY-MM-DD HH:mm:ss' @@ -23,7 +23,7 @@ export default function CurrentTableStatus({
-
Data quality KPI score:
+
Data quality KPI score
{tableDataQualityStatus.data_quality_kpi !== undefined ? Number(tableDataQualityStatus.data_quality_kpi).toFixed(2) + ' %' diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatus.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatus.tsx index 0cf25866f6..3787c4fb61 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatus.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatus.tsx @@ -64,6 +64,9 @@ export default function TableQualityStatus({ const [severityType, setSeverityType] = useState<'current' | 'highest'>( 'current' ); + const [extendedChecks, setExtendedChecks] = useState< + Array<{ checkType: string; categoryDimension: string }> + >([]); const [month, setMonth] = useState(); const [since, setSince] = useState( new Date(moment().subtract(30, 'days').format('YYYY-MM-DD')) @@ -160,6 +163,22 @@ export default function TableQualityStatus({ onChangeFirstLevelChecks(); }, [categoryDimension, tableDataQualityStatus, severityType]); + const handleCategoryDimensionChange = (value: 'category' | 'dimension') => { + setCategoryDimension(value); + setExtendedChecks([]); + }; + + const handleSeverityTypeChange = (value: 'current' | 'highest') => { + setSeverityType(value); + setExtendedChecks([]); + }; + + const handleMonthChange = (value: number | undefined) => { + setMonth(value); + setSince(undefined); + setExtendedChecks([]); + }; + return (
{timePartitioned && @@ -184,47 +203,45 @@ export default function TableQualityStatus({ label="category" fontClassName="text-sm" className="!items-start" - onClick={() => setCategoryDimension('category')} + onClick={() => handleCategoryDimensionChange('category')} /> setCategoryDimension('dimension')} + onClick={() => handleCategoryDimensionChange('dimension')} />
{ - setSince(undefined), setMonth(1); - }} + onClick={() => handleMonthChange(1)} fontClassName="text-sm" /> { - setSince(undefined), setMonth(3); - }} + onClick={() => handleMonthChange(3)} fontClassName="text-sm" /> { - setMonth(undefined); - }} + onClick={() => handleMonthChange(undefined)} fontClassName="text-sm" /> { + setSince(date); + setExtendedChecks([]); + }} selected={since} dateFormat="yyyy-MM-dd" + disabled={month !== undefined} />
@@ -232,13 +249,13 @@ export default function TableQualityStatus({ setSeverityType('current')} + onClick={() => handleSeverityTypeChange('current')} fontClassName="text-sm" /> setSeverityType('highest')} + onClick={() => handleSeverityTypeChange('highest')} fontClassName="text-sm" />
@@ -254,6 +271,8 @@ export default function TableQualityStatus({ severityType={severityType} tableDataQualityStatus={tableDataQualityStatus} timeScale={timePartitioned} + extendedChecks={extendedChecks} + setExtendedChecks={setExtendedChecks} /> ) : (
No data quality check results
diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusCategory.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusCategory.tsx index 1394a1cb30..0ce9ffa9eb 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusCategory.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusCategory.tsx @@ -91,8 +91,10 @@ export default function TableQualityStatusCategory({ style={{ padding: '0.5px', paddingBottom: 0, margin: '0.5px' }} >
tableStatus.status && toggleExtendedChecks(key, 'table')} + className="h-full flex w-29 items-center justify-end relative" + onClick={() => + tableStatus.status && toggleExtendedChecks(key, 'table') + } >
{tableStatus.status && ( @@ -108,32 +110,8 @@ export default function TableQualityStatusCategory({ : {} } > - {showTooltip && ( - -
- - )} {showIcon && ( -
+
)} + {showTooltip && ( + +
+ + )}
)}
@@ -209,7 +211,7 @@ export default function TableQualityStatusCategory({ )} style={{ fontSize: '11px', - ...(getColor(severity) === 'bg-gray-150' + ...(getSecondColor(severity) === 'bg-gray-150' ? secondBackgroundStyle : {}) }} diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusColumnCategory.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusColumnCategory.tsx index dc18608dfd..a4bc9ef646 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusColumnCategory.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusColumnCategory.tsx @@ -134,17 +134,14 @@ export default function TableQualityStatusColumnCategory({ const showTooltip = columnCircleStatus.lastExecutedAt !== null; const showIcon = columnStatus.status !== null; - const isExtended = extendedChecks.some( - (x) => - x.checkType === customKey && - x.categoryDimension === firstLevelChecksKey - ); - return ( columnStatus.status && toggleExtendedChecks(customKey, firstLevelChecksKey)} + onClick={() => + columnStatus.status && + toggleExtendedChecks(customKey, firstLevelChecksKey) + } style={{ padding: 0, margin: 0 }} > {columnStatus.status && ( @@ -157,7 +154,7 @@ export default function TableQualityStatusColumnCategory({
- {showTooltip && ( - -
- - )} {showIcon && ( -
+
)} + {showTooltip && ( + +
+ + )}
)} @@ -262,7 +259,7 @@ export default function TableQualityStatusColumnCategory({ >
; + setExtendedChecks: React.Dispatch< + React.SetStateAction< + Array<{ checkType: string; categoryDimension: string }> + > + >; } export const severityMap = [ diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusOverview.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusOverview.tsx index 998437d171..5805b1203c 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusOverview.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusOverview.tsx @@ -1,6 +1,6 @@ import clsx from 'clsx'; import moment from 'moment'; -import React, { useState } from 'react'; +import React from 'react'; import TableQualityStatusCategory from './TableQualityStatusCategory'; import TableQualityStatusColumnCategory from './TableQualityStatusColumnCategory'; import { @@ -13,56 +13,10 @@ export default function TableQualityStatusOverview({ categoryDimension, severityType, tableDataQualityStatus, - timeScale + timeScale, + extendedChecks, + setExtendedChecks }: ITableParameters) { - const [extendedChecks, setExtendedChecks] = useState< - Array<{ checkType: string; categoryDimension: string }> - >([]); - // const dispatch = useActionDispatch(); - // const history = useHistory(); - // const { - // checkTypes, - // connection, - // schema, - // table - // }: { - // checkTypes: CheckTypes; - // connection: string; - // schema: string; - // table: string; - // } = useDecodedParams(); - - // const openFirstLevelTableTab = ( - // checkTypes: CheckTypes, - // connection: string, - // schema: string, - // table: string, - // timeScale?: 'daily' | 'monthly' - // ) => { - // const url = ROUTES.TABLE_LEVEL_PAGE( - // checkTypes, - // connection, - // schema, - // table, - // timeScale ?? 'advanced' - // ); - // const value = ROUTES.TABLE_LEVEL_VALUE( - // checkTypes, - // connection, - // schema, - // table - // ); - // dispatch( - // addFirstLevelTab(checkTypes, { - // url, - // value, - // state: {}, - // label: table - // }) - // ); - // history.push(url); - // }; - const renderTooltipContent = (lastExecutedAt: any, severity: any) => { return (
@@ -131,15 +85,6 @@ export default function TableQualityStatusOverview({ className={clsx( ' border-b border-gray-300 font-bold text-xs text-center' )} - // onClick={() => - // openFirstLevelTableTab( - // checkTypes, - // connection, - // schema, - // table, - // timeScale - // ) - // } >
{key?.replaceAll('_', ' ').replace(/./, (c) => c.toUpperCase())} diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusUtils.ts b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusUtils.ts index 7d7f178467..5fddbccf4e 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusUtils.ts +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TableQualityStatusUtils.ts @@ -18,13 +18,13 @@ export const getColor = ( case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.execution_error: return 'bg-gray-150'; case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.fatal: - return 'bg-red-300'; + return 'bg-[#EF8079]'; case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.error: - return 'bg-orange-300'; + return 'bg-[#EFB87E]'; case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.warning: - return 'bg-yellow-300'; + return 'bg-[#EFEC82]'; case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.valid: - return 'bg-green-300'; + return 'bg-[#5CBCAB]'; default: return ''; } @@ -38,8 +38,6 @@ export const getSecondColor = ( ) => { // console.log(status) switch (status) { - case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.execution_error: - return 'bg-gray-150'; case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.fatal: return 'bg-[#FEEDEC]'; case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.error: @@ -48,6 +46,29 @@ export const getSecondColor = ( return 'bg-[#FDFDED]'; case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.valid: return 'bg-[#DDF2EF]'; + default: + return 'bg-gray-150'; + } +}; +export const getDimensionColor = ( + status: + | CheckCurrentDataQualityStatusModelCurrentSeverityEnum + | DimensionCurrentDataQualityStatusModelCurrentSeverityEnum + | null + | undefined +) => { + // console.log(status) + switch (status) { + case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.execution_error: + return 'bg-gray-150'; + case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.fatal: + return 'bg-[#E3170A]'; + case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.error: + return 'bg-[#FF9900]'; + case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.warning: + return 'bg-[#EBE51E]'; + case CheckCurrentDataQualityStatusModelCurrentSeverityEnum.valid: + return 'bg-[#029A80]'; default: return ''; } diff --git a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TotalChecksExecuted.tsx b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TotalChecksExecuted.tsx index 17c1df3cc9..4c8b8a1bda 100644 --- a/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TotalChecksExecuted.tsx +++ b/dqops/src/main/frontend/src/components/Connection/TableView/TableQualityStatus/TotalChecksExecuted.tsx @@ -10,27 +10,27 @@ export default function TotalChecksExecuted({ return (
-
Total checks executed:
+
Total checks executed
{tableDataQualityStatus.executed_checks}
-
Correct results:
+
Correct results
{tableDataQualityStatus.valid_results}
-
Warnings:
+
Warnings
{tableDataQualityStatus.warnings}
-
Errors:
+
Errors
{tableDataQualityStatus.errors}
-
Fatal results:
+
Fatal results
{tableDataQualityStatus.fatals}
-
Execution errors:
+
Execution errors
{tableDataQualityStatus.execution_errors}
 diff --git a/dqops/src/main/frontend/src/components/CustomTree/CollectStatisticsDialog.tsx b/dqops/src/main/frontend/src/components/CustomTree/CollectStatisticsDialog.tsx index 7e07a19f6c..a6700d073c 100644 --- a/dqops/src/main/frontend/src/components/CustomTree/CollectStatisticsDialog.tsx +++ b/dqops/src/main/frontend/src/components/CustomTree/CollectStatisticsDialog.tsx @@ -5,17 +5,17 @@ import { DialogHeader } from '@material-tailwind/react'; import React, { useState } from 'react'; -import Button from '../Button'; +import { useSelector } from 'react-redux'; import { StatisticsCollectorSearchFilters, StatisticsCollectorSearchFiltersTargetEnum } from '../../api'; -import Input from '../Input'; -import LabelsView from '../Connection/LabelsView'; -import SelectInput from '../SelectInput'; -import { useSelector } from 'react-redux'; import { IRootState } from '../../redux/reducers'; +import Button from '../Button'; +import LabelsView from '../Connection/LabelsView'; import SectionWrapper from '../Dashboard/SectionWrapper'; +import Input from '../Input'; +import SelectInput from '../SelectInput'; import SvgIcon from '../SvgIcon'; type TCollectStatisticsDialogProps = { onClick: (node: StatisticsCollectorSearchFilters) => void; @@ -95,7 +95,7 @@ export default function CollectStatisticsDialog({
{additionalParams === false ? (
setAdditionalParams(true)} > @@ -106,7 +106,7 @@ export default function CollectStatisticsDialog({ title="Additional parameters" onClick={() => setAdditionalParams(false)} svgIcon={true} - className="cursor-default" + titleClassName="cursor-pointer" >
diff --git a/dqops/src/main/frontend/src/components/CustomTree/ContextMenu.tsx b/dqops/src/main/frontend/src/components/CustomTree/ContextMenu.tsx index 9681a4e4ec..cab57c8d68 100644 --- a/dqops/src/main/frontend/src/components/CustomTree/ContextMenu.tsx +++ b/dqops/src/main/frontend/src/components/CustomTree/ContextMenu.tsx @@ -389,7 +389,7 @@ const ContextMenu = ({ node }: ContextMenuProps) => { : undefined; }} > - Delete data quality results + Delete data
)} @@ -404,7 +404,7 @@ const ContextMenu = ({ node }: ContextMenuProps) => { : undefined; }} > - Delete data quality results + Delete data
)} diff --git a/dqops/src/main/frontend/src/components/CustomTree/DeleteOnlyDataDialog.tsx b/dqops/src/main/frontend/src/components/CustomTree/DeleteOnlyDataDialog.tsx index 9aa5892f80..2b1195aa92 100644 --- a/dqops/src/main/frontend/src/components/CustomTree/DeleteOnlyDataDialog.tsx +++ b/dqops/src/main/frontend/src/components/CustomTree/DeleteOnlyDataDialog.tsx @@ -82,7 +82,7 @@ const DeleteOnlyDataDialog = ({ return ( - Delete data quality results + Delete data
{hierarchiArray?.[0] && ( diff --git a/dqops/src/main/frontend/src/components/CustomTree/DeleteStoredDataExtendedPopUp.tsx b/dqops/src/main/frontend/src/components/CustomTree/DeleteStoredDataExtendedPopUp.tsx index 426e0398c5..04da1d3820 100644 --- a/dqops/src/main/frontend/src/components/CustomTree/DeleteStoredDataExtendedPopUp.tsx +++ b/dqops/src/main/frontend/src/components/CustomTree/DeleteStoredDataExtendedPopUp.tsx @@ -70,7 +70,7 @@ const DeleteStoredDataExtendedPopUp = ({ const [allChecks, setAllChecks] = useState(); const [basicStatisticsFiltersOpen, setBasicStatisticsFiltersOpen] = - useState(true); + useState(false); const { userProfile } = useSelector((state: IRootState) => state.job || {}); const toUTCString = (date: Date) => moment(date).utc().format('YYYY-MM-DD'); @@ -170,7 +170,7 @@ const DeleteStoredDataExtendedPopUp = ({ return ( - Delete data quality results + Delete data
@@ -438,7 +438,7 @@ const DeleteStoredDataExtendedPopUp = ({ } />
-
+
@@ -482,6 +482,14 @@ const DeleteStoredDataExtendedPopUp = ({ label="Error samples" checkClassName="bg-teal-500" /> + + onChangeParams({ deleteChecksConfiguration }) + } + label="Data quality checks configuration" + checkClassName="bg-teal-500" + />
diff --git a/dqops/src/main/frontend/src/components/CustomTree/RunChecksDialog.tsx b/dqops/src/main/frontend/src/components/CustomTree/RunChecksDialog.tsx index 9e78e8c07a..034beb843a 100644 --- a/dqops/src/main/frontend/src/components/CustomTree/RunChecksDialog.tsx +++ b/dqops/src/main/frontend/src/components/CustomTree/RunChecksDialog.tsx @@ -22,7 +22,9 @@ import Button from '../Button'; import Checkbox from '../Checkbox'; import LabelsView from '../Connection/LabelsView'; import SectionWrapper from '../Dashboard/SectionWrapper'; +import DatePicker from '../DatePicker'; import Input from '../Input'; +import RadioButton from '../RadioButton'; import Select from '../Select'; import SelectInput from '../SelectInput'; import SvgIcon from '../SvgIcon'; @@ -63,7 +65,7 @@ export default function RunChecksDialog({ from_date: moment().utc().subtract(7, 'days').format('YYYY-MM-DD'), to_date: moment().utc().format('YYYY-MM-DD') }); - + const [timeWindowPartitioned, setTimeWindowPartitioned] = useState(true); const [additionalParams, setAdditionalParams] = useState(false); const onChangeFilters = ( @@ -120,43 +122,6 @@ export default function RunChecksDialog({ Run checks - {checkType === CheckTypes.PARTITIONED && ( -
-
- From - - onChangeTimeFilterWindow({ from_date: e.target.value }) - } - className={clsx( - 'mt-2', - !isDateValid(timeWindowFilter?.from_date) - ? 'border border-red-500' - : '' - )} - placeholder="*" - /> -
-
- To - - onChangeTimeFilterWindow({ to_date: e.target.value }) - } - className={clsx( - 'mt-2', - !isDateValid(timeWindowFilter?.to_date) - ? 'border border-red-500' - : '' - )} - placeholder="*" - /> -
-
-
- )}
Connection @@ -181,7 +146,7 @@ export default function RunChecksDialog({
-
+
Column name
-
+
Column datatype
-
- Column nullable +
onChangeFilters({ columnNullable: value })} + className="mr-4" /> + Column nullable +
+
+ + onChangeFilters({ collectErrorSample: value }) + } + className="mr-4" + /> + Collect error samples
- {checkType === CheckTypes.PARTITIONED && ( -
- ({ label: key, value: key }) + ) + ]} + value={filters.timeWindowFilter} + onChange={(value) => + onChangeFilters({ timeWindowFilter: value }) + } + // label="Time window for partitioned checks" + disabled={ + filters.checkType !== CheckTypes.PARTITIONED || + !timeWindowPartitioned + } + triggerClassName={clsx( + filters.checkType !== CheckTypes.PARTITIONED || + !timeWindowPartitioned + ? 'text-gray-500' + : '' + )} + menuClassName="!top-9 !max-h-29 !z-50" + /> +
+
+
+ setTimeWindowPartitioned(false)} + className="mr-6" + disabled={filters.checkType !== CheckTypes.PARTITIONED} + /> +
For the time range
+
+ + onChangeTimeFilterWindow({ + from_date: moment(e).utc().format('YYYY-MM-DD') + }) + } + className={clsx( + 'border border-gray-300', + !isDateValid(timeWindowFilter?.from_date) + ? 'border border-red-500' + : '' + )} + dateFormat="yyyy-MM-dd" + disabled={ + filters.checkType !== CheckTypes.PARTITIONED || + timeWindowPartitioned + } + /> +
+
to
+
+ + onChangeTimeFilterWindow({ + to_date: moment(e).utc().format('YYYY-MM-DD') + }) + } + className={clsx( + 'border border-gray-300', + !isDateValid(timeWindowFilter?.to_date) + ? 'border border-red-500' + : '' + )} + disabled={ + filters.checkType !== CheckTypes.PARTITIONED || + timeWindowPartitioned + } + dateFormat="yyyy-MM-dd" + /> +
+
+
+
+
+ + onChangeTimeFilterWindow({ where_filter: e.target.value }) + } + /> +
)} @@ -370,10 +440,15 @@ export default function RunChecksDialog({ isSaveEnabled ? (onClick( prepareFilters(filters), - (filters.timeWindowFilter && - checkType === CheckTypes.PARTITIONED - ? RUN_CHECK_TIME_WINDOW_FILTERS[filters.timeWindowFilter] - : timeWindowFilter) ?? undefined, + checkType === CheckTypes.PARTITIONED && timeWindowPartitioned + ? { + ...RUN_CHECK_TIME_WINDOW_FILTERS[ + filters.timeWindowFilter ?? + 'Default incremental time window' + ], + where_filter: timeWindowFilter?.where_filter + } + : timeWindowFilter, filters.collectErrorSample ), setFilters(runChecksJobTemplate)) diff --git a/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/DuckDBConnection/index.tsx b/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/DuckDBConnection/index.tsx index 045a6c3dcf..11b40bb4d0 100644 --- a/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/DuckDBConnection/index.tsx +++ b/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/DuckDBConnection/index.tsx @@ -1,8 +1,5 @@ -import React, { useEffect, useState } from 'react'; +import React, { useState } from 'react'; -import { cloneDeep } from 'lodash'; -import { useSelector } from 'react-redux'; -import { useParams } from 'react-router-dom'; import { DuckdbParametersSpec, DuckdbParametersSpecAwsAuthenticationModeEnum, @@ -12,8 +9,6 @@ import { SharedCredentialListModel } from '../../../../api'; import { TConfiguration } from '../../../../components/FileFormatConfiguration/TConfiguration'; -import { getFirstLevelActiveTab } from '../../../../redux/selectors'; -import { CheckTypes } from '../../../../shared/routes'; import FieldTypeInput from '../../../Connection/ConnectionView/FieldTypeInput'; import FileFormatConfiguration from '../../../FileFormatConfiguration/FileFormatConfiguration'; import KeyValueProperties from '../../../FileFormatConfiguration/KeyValueProperties'; @@ -96,53 +91,41 @@ const DuckdbConnection = ({ sharedCredentials, freezeFileType = false }: IDuckdbConnectionProps) => { - const [copiedDatabase, setCopiedDatabase] = useState( - cloneDeep(duckdb) ?? {} - ); - const { checkTypes }: { checkTypes: CheckTypes } = useParams(); - const firstLevelActiveTab = useSelector(getFirstLevelActiveTab(checkTypes)); const [fileFormatType, setFileFormatType] = useState< keyof DuckdbParametersSpec >(duckdb?.files_format_type ?? DuckdbParametersSpecFilesFormatTypeEnum.csv); - const [refetchDirectoriesIndicator, setRefetchDirectoriesIndicator] = - useState(false); - const [selectedInput, setSelectedInput] = useState(); + const [selectedInput, setSelectedInput] = useState(); + const handleChange = (obj: Partial) => { + onChange({ + ...duckdb, + ...obj + }); + }; const onChangeConfiguration = (params: Partial) => { - setCopiedDatabase((prev) => ({ - ...prev, + handleChange({ [fileFormatType]: { - ...(prev?.[fileFormatType] as TConfiguration), + ...(duckdb?.[fileFormatType] as TConfiguration), ...params } - })); + }); }; const cleanConfiguration = () => {}; const onChangeFile = (val: DuckdbParametersSpecFilesFormatTypeEnum) => { - setCopiedDatabase((prev) => ({ - ...prev, + handleChange({ [fileFormatType as keyof DuckdbParametersSpec]: undefined, [val as keyof DuckdbParametersSpec]: {}, files_format_type: val - })); + }); setFileFormatType(val); cleanConfiguration(); }; - useEffect(() => { - onChange(cloneDeep(copiedDatabase) ?? {}); - }, [copiedDatabase]); - - useEffect(() => { - setCopiedDatabase(cloneDeep(duckdb) ?? {}); - setRefetchDirectoriesIndicator((prev) => !prev); - }, [firstLevelActiveTab]); - const changeStorageTypeDirectoryPrefixes = ( storage_type: DuckdbParametersSpecStorageTypeEnum ) => { - const directories = { ...copiedDatabase?.directories }; + const directories = { ...duckdb?.directories }; Object.keys(directories ?? {}).forEach((key) => { if ( @@ -164,7 +147,7 @@ const DuckdbConnection = ({ directories[key] = ''; } }); - setCopiedDatabase((prev) => ({ ...prev, directories, storage_type })); + handleChange({ directories, storage_type }); }; const awsStorageForm = (): JSX.Element => { @@ -174,70 +157,61 @@ const DuckdbConnection = ({ label="AWS authentication mode" options={awsAuthenticationOptions} className="mb-4" - value={copiedDatabase?.aws_authentication_mode} + value={duckdb?.aws_authentication_mode} onChange={(value) => { - setCopiedDatabase((prev) => ({ - ...prev, + handleChange({ aws_authentication_mode: value - })); + }); }} onClickValue={setSelectedInput} selectedMenu={selectedInput} menuClassName="!top-14" /> - {copiedDatabase?.aws_authentication_mode === + {duckdb?.aws_authentication_mode === DuckdbParametersSpecAwsAuthenticationModeEnum.iam && ( <> - setCopiedDatabase((prev) => ({ ...prev, user: value })) - } + value={duckdb?.user} + onChange={(value) => handleChange({ user: value })} /> - setCopiedDatabase((prev) => ({ ...prev, password: value })) - } + value={duckdb?.password} + onChange={(value) => handleChange({ password: value })} /> - setCopiedDatabase((prev) => ({ ...prev, region: value })) - } + value={duckdb?.region} + onChange={(value) => handleChange({ region: value })} /> )} - {copiedDatabase?.aws_authentication_mode === + {duckdb?.aws_authentication_mode === DuckdbParametersSpecAwsAuthenticationModeEnum.default_credentials && ( - setCopiedDatabase((prev) => ({ ...prev, region: value })) - } + onChange={(value) => handleChange({ region: value })} /> )} @@ -251,18 +225,15 @@ const DuckdbConnection = ({ label="Azure authentication mode" options={azureAuthenticationOptions} className="mb-4" - value={copiedDatabase?.azure_authentication_mode} + value={duckdb?.azure_authentication_mode} onChange={(value) => { - setCopiedDatabase((prev) => ({ - ...prev, - azure_authentication_mode: value - })); + handleChange({ azure_authentication_mode: value }); }} onClickValue={setSelectedInput} selectedMenu={selectedInput} /> - {copiedDatabase?.azure_authentication_mode === + {duckdb?.azure_authentication_mode === DuckdbParametersSpecAzureAuthenticationModeEnum.connection_string && ( <> - setCopiedDatabase((prev) => ({ ...prev, password: value })) - } + value={duckdb?.password} + onChange={(value) => handleChange({ password: value })} /> )} - {copiedDatabase?.azure_authentication_mode === + {duckdb?.azure_authentication_mode === DuckdbParametersSpecAzureAuthenticationModeEnum.credential_chain && ( <> - setCopiedDatabase((prev) => ({ ...prev, account_name: value })) - } + value={duckdb?.account_name} + onChange={(value) => handleChange({ account_name: value })} /> )} - {copiedDatabase?.azure_authentication_mode === + {duckdb?.azure_authentication_mode === DuckdbParametersSpecAzureAuthenticationModeEnum.service_principal && ( <> - setCopiedDatabase((prev) => ({ ...prev, tenant_id: value })) - } + value={duckdb?.tenant_id} + onChange={(value) => handleChange({ tenant_id: value })} /> - setCopiedDatabase((prev) => ({ ...prev, client_id: value })) - } + value={duckdb?.client_id} + onChange={(value) => handleChange({ client_id: value })} /> - setCopiedDatabase((prev) => ({ ...prev, client_secret: value })) - } + value={duckdb?.client_secret} + onChange={(value) => handleChange({ client_secret: value })} /> - setCopiedDatabase((prev) => ({ ...prev, account_name: value })) - } + value={duckdb?.account_name} + onChange={(value) => handleChange({ account_name: value })} /> )} @@ -346,20 +305,16 @@ const DuckdbConnection = ({ data={sharedCredentials} label="Access Key" className="mb-4 text-sm" - value={copiedDatabase?.user} - onChange={(value) => - setCopiedDatabase((prev) => ({ ...prev, user: value })) - } + value={duckdb?.user} + onChange={(value) => handleChange({ user: value })} /> - setCopiedDatabase((prev) => ({ ...prev, password: value })) - } + value={duckdb?.password} + onChange={(value) => handleChange({ password: value })} /> ); @@ -374,21 +329,21 @@ const DuckdbConnection = ({ label="Files location" options={storageTypeOptions} className="mb-4 text-sm" - value={copiedDatabase?.storage_type} + value={duckdb?.storage_type} onChange={changeStorageTypeDirectoryPrefixes} onClickValue={setSelectedInput} selectedMenu={selectedInput} menuClassName="!top-14" /> - {copiedDatabase?.storage_type === - DuckdbParametersSpecStorageTypeEnum.s3 && awsStorageForm()} + {duckdb?.storage_type === DuckdbParametersSpecStorageTypeEnum.s3 && + awsStorageForm()} - {copiedDatabase?.storage_type === - DuckdbParametersSpecStorageTypeEnum.azure && azureStorageForm()} + {duckdb?.storage_type === DuckdbParametersSpecStorageTypeEnum.azure && + azureStorageForm()} - {copiedDatabase?.storage_type === - DuckdbParametersSpecStorageTypeEnum.gcs && googleStorageForm()} + {duckdb?.storage_type === DuckdbParametersSpecStorageTypeEnum.gcs && + googleStorageForm()} { - setCopiedDatabase((prev) => ({ - ...prev, - directories: directories - })); + handleChange({ directories }); }} sharedCredentials={sharedCredentials} - storageType={copiedDatabase?.storage_type} - refetchDirectoriesIndicator={refetchDirectoriesIndicator} + storageType={duckdb?.storage_type} /> - setCopiedDatabase((prev) => ({ ...prev, properties })) - } + properties={duckdb?.properties} + onChange={(properties) => handleChange({ properties })} sharedCredentials={sharedCredentials} /> diff --git a/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/JdbcProperties/JdbcPropertyItem.tsx b/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/JdbcProperties/JdbcPropertyItem.tsx index a389ab9b8f..d7d72f0674 100644 --- a/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/JdbcProperties/JdbcPropertyItem.tsx +++ b/dqops/src/main/frontend/src/components/Dashboard/DatabaseConnection/JdbcProperties/JdbcPropertyItem.tsx @@ -67,6 +67,7 @@ const JdbcPropertyItem = ({ {!(index === 0 && properties.length === 1) && ( void; sharedCredentials?: SharedCredentialListModel[]; + title?: string; } + function convertObjectToArray(obj: { [key: string]: string; }): { [key: string]: string }[] { @@ -30,27 +32,42 @@ function convertArrayToObject(array: { [key: string]: string }[]): { }, {}); } +function arePropertiesEqual(obj1: IProperties, obj2: IProperties): boolean { + const keys1 = Object.keys(obj1); + const keys2 = Object.keys(obj2); + + if (keys1.length !== keys2.length) return false; + + for (const key of keys1) { + if (obj1[key] !== obj2[key]) return false; + } + + return true; +} + const JdbcPropertiesView = ({ - properties = {[''] : ''}, + properties = { ['']: '' }, onChange, - sharedCredentials + sharedCredentials, + title }: IJdbcPropertiesViewProps) => { - const [arr, setArr] = useState(convertObjectToArray(properties ?? {})); + const [arr, setArr] = useState( + convertObjectToArray(properties ?? { ['']: '' }) + ); - const onChangeArr = ( - array: { - [key: string]: string; - }[] - ) => { + const onChangeArr = (array: { [key: string]: string }[]) => { setArr(array); onChange(convertArrayToObject(array)); }; - // useEffect(() => { - // if (properties) { - // setArr(convertObjectToArray(properties ?? {})); - // } - // }, [firstLevelActiveTab, storageType]); + useEffect(() => { + if (!arePropertiesEqual(convertArrayToObject(arr), properties)) { + setArr(convertObjectToArray(properties ?? { ['']: '' })); + } + if (Object.keys(properties).length === 0 || properties === undefined) { + setArr([{ ['']: '' }]); + } + }, [properties]); return (
@@ -58,7 +75,7 @@ const JdbcPropertiesView = ({ - JDBC connection property + {title ?? 'JDBC connection property'} Value Action diff --git a/dqops/src/main/frontend/src/components/Dashboard/SectionWrapper/index.tsx b/dqops/src/main/frontend/src/components/Dashboard/SectionWrapper/index.tsx index 32d756334a..362726d2cb 100644 --- a/dqops/src/main/frontend/src/components/Dashboard/SectionWrapper/index.tsx +++ b/dqops/src/main/frontend/src/components/Dashboard/SectionWrapper/index.tsx @@ -5,7 +5,7 @@ import SvgIcon from '../../SvgIcon'; interface ISectionWrapperProps { title: string; - titleIcon?: React.ReactNode + titleIcon?: React.ReactNode; children?: any; className?: string; svgIcon?: boolean; @@ -32,11 +32,14 @@ const SectionWrapper = ({ className={clsx( 'px-2 absolute bg-white left-2 top-0 -translate-y-1/2 text-gray-700 font-semibold', svgIcon ? 'flex items-center justify-center' : '', + onClick ? 'cursor-pointer' : '', titleClassName )} onClick={onClick} > - {svgIcon && } + {svgIcon && ( + + )} {title}
{children} diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckCategoriesView.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckCategoriesView.tsx index 8de293455b..b653350a3c 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckCategoriesView.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckCategoriesView.tsx @@ -4,6 +4,7 @@ import { useSelector } from 'react-redux'; import { CheckModel, CheckResultsOverviewDataModel, + DqoJobHistoryEntryModelStatusEnum, QualityCategoryModel, TimeWindowFilterParameters } from '../../api'; @@ -57,14 +58,17 @@ const CheckCategoriesView = ({ ruleParamenterConfigured, onChangeRuleParametersConfigured }: CheckCategoriesViewProps) => { + const { job_dictionary_state, userProfile } = useSelector( + (state: IRootState) => state.job || {} + ); + const [jobId, setJobId] = useState(); + const job = jobId ? job_dictionary_state[jobId] : undefined; const [deleteDataDialogOpened, setDeleteDataDialogOpened] = useState(false); const { checkTypes }: { checkTypes: CheckTypes } = useDecodedParams(); const dispatch = useActionDispatch(); const firstLevelActiveTab = useSelector(getFirstLevelActiveTab(checkTypes)); const [isExtended, setIsExtended] = useState(false); - const { userProfile } = useSelector((state: IRootState) => state.job || {}); - const shouldExtend = () => { if ( category.checks?.some( @@ -93,16 +97,23 @@ const CheckCategoriesView = ({ res.data?.jobId?.jobId ?? 0 ) ); - - if (getCheckOverview) { - getCheckOverview(); - } + setJobId(res.data?.jobId?.jobId); }; useEffect(() => { shouldExtend(); }, []); + useEffect(() => { + if (!getCheckOverview) return; + if ( + job?.status === DqoJobHistoryEntryModelStatusEnum.finished || + job?.status === DqoJobHistoryEntryModelStatusEnum.failed + ) { + getCheckOverview(); + } + }, [job?.status]); + return ( @@ -235,6 +246,8 @@ const CheckCategoriesView = ({ JobApiClient.deleteStoredData(undefined, false, undefined, { ...category.data_clean_job_template, ...params + }).then((res) => { + setJobId(res.data?.jobId?.jobId); }); }} /> diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckDetails.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckDetails.tsx index e897009c77..e10f27c30a 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckDetails.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckDetails.tsx @@ -334,10 +334,15 @@ const CheckDetails = ({ }; const refetch = (month: string, name?: string) => { - fetchCheckErrors(month, name); - fetchCheckResults(month, name); - fetchCheckReadouts(month, name); - fetchErrorSamples(month, name); + if (activeTab === 'check_results') { + fetchCheckResults(month, name); + } else if (activeTab === 'sensor_readouts') { + fetchCheckReadouts(month, name); + } else if (activeTab === 'execution_errors') { + fetchCheckErrors(month, name); + } else if (activeTab === 'error_sampling') { + fetchErrorSamples(month, name); + } }; useEffect(() => { @@ -350,15 +355,7 @@ const CheckDetails = ({ }, [fetchCheckResults]); useEffect(() => { - if (activeTab === 'check_results') { - fetchCheckResults(filters.month, filters.dataGroup); - } else if (activeTab === 'sensor_readouts') { - fetchCheckReadouts(filters.month, filters.dataGroup); - } else if (activeTab === 'execution_errors') { - fetchCheckErrors(filters.month, filters.dataGroup); - } else if (activeTab === 'error_sampling') { - fetchErrorSamples(filters.month, filters.dataGroup); - } + refetch(filters.month, filters.dataGroup); }, [activeTab]); const tabs = [ diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckErrorsTab.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckErrorsTab.tsx index 1ac1c97975..7215276008 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckErrorsTab.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckErrorsTab.tsx @@ -3,9 +3,9 @@ import React, { useMemo } from 'react'; import { ErrorsListModel } from '../../../api'; import { useTree } from '../../../contexts/treeContext'; import { getLocalDateInUserTimeZone } from '../../../utils'; +import SelectTailwind from '../../Select/SelectTailwind'; import { Table } from '../../Table'; import ErrorText from './ErrorText'; -import SelectTailwind from '../../Select/SelectTailwind'; interface CheckErrorsTabProps { errors: ErrorsListModel[]; @@ -26,25 +26,25 @@ const CheckErrorsTab = ({ const columns = [ { - label: 'Executed At', + label: 'Executed at', value: 'executedAt', className: 'text-sm !py-2 whitespace-nowrap text-gray-700 w-60' }, { - label: 'Error Source', + label: 'Error source', value: 'errorSource', className: 'text-sm !py-2 whitespace-nowrap text-gray-700 w-50' }, { - label: 'Error Message', + label: 'Error message', value: 'errorMessage', className: 'text-sm !py-2 text-gray-700 w-120', render: (text: string) => }, { - label: 'Readout Id', + label: 'Readout id', value: 'readoutId', - className: 'text-sm !py-2 whitespace-nowrap text-gray-700 w-80 text-right' + className: 'text-sm !py-2 whitespace-nowrap text-gray-700 w-80 text-left' } ]; @@ -65,7 +65,7 @@ const CheckErrorsTab = ({ return (
@@ -80,6 +80,7 @@ const CheckErrorsTab = ({ })) || [] } onChange={onChangeDataGroup} + disabled={(errors[0]?.dataGroupsNames ?? []).length === 0} />
@@ -95,18 +96,19 @@ const CheckErrorsTab = ({
No Data
)} {errors.map((result, index) => ( -
+
({ ...item, checkName: result.checkName, executedAt: moment( getLocalDateInUserTimeZone(new Date(String(item.executedAt))) - ).format('YYYY-MM-DD HH:mm:ss') + ).format('YYYY-MM-DD HH:mm:ss'), + timePeriod: item.timePeriod?.replace(/T/g, ' ') }))} - emptyMessage="No Data" + emptyMessage="No data" /> ))} diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckResultsTab.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckResultsTab.tsx index 4995badbb8..85152f989e 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckResultsTab.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/CheckResultsTab.tsx @@ -74,23 +74,23 @@ const CheckResultsTab = ({ checkTypes === 'profiling' ? 'Profile date (local time)' : checkTypes === 'partitioned' - ? 'Partition Date' + ? 'Partition date' : 'Checkpoint date', value: 'timePeriod', className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700' }, { - label: 'Time Scale', + label: 'Time scale', value: 'timeGradient', className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700' }, { - label: 'Executed At', + label: 'Executed at', value: 'executedAt', className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700' }, { - label: 'Actual Value', + label: 'Actual value', value: 'actualValue', className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700 text-right', @@ -99,7 +99,7 @@ const CheckResultsTab = ({ ) }, { - label: 'Expected Value', + label: 'Expected value', value: 'expectedValue', className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700 text-right', @@ -112,7 +112,7 @@ const CheckResultsTab = ({ Issue
- Severity Level + severity level ), value: 'severity', @@ -144,7 +144,7 @@ const CheckResultsTab = ({ Warning
- Lower Threshold + lower threshold ), value: 'warningLowerBound', @@ -159,7 +159,7 @@ const CheckResultsTab = ({ Warning
- Upper Threshold + upper threshold ), value: 'warningUpperBound', @@ -174,7 +174,7 @@ const CheckResultsTab = ({ Error
- Lower Threshold + lower threshold ), value: 'errorLowerBound', @@ -189,7 +189,7 @@ const CheckResultsTab = ({ Error
- Upper Threshold + upper threshold ), value: 'errorUpperBound', @@ -204,7 +204,7 @@ const CheckResultsTab = ({ Fatal
- Lower Threshold + lower threshold ), value: 'fatalLowerBound', @@ -219,7 +219,7 @@ const CheckResultsTab = ({ Fatal
- Upper Threshold + upper threshold ), value: 'fatalUpperBound', @@ -230,19 +230,19 @@ const CheckResultsTab = ({ ) }, { - label: 'Duration Ms', + label: 'Duration ms', value: 'durationMs', className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700 text-right' }, { - label: 'Data Group', + label: 'Data group', value: 'dataGroup', className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700 text-left' }, { label: 'Id', value: 'id', - className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700 text-right' + className: 'text-sm px-4 !py-2 whitespace-nowrap text-gray-700 text-left' } ]; @@ -303,7 +303,7 @@ const CheckResultsTab = ({ return (
@@ -318,6 +318,7 @@ const CheckResultsTab = ({ })) || [] } onChange={onChangeDataGroup} + disabled={(results[0]?.dataGroups ?? []).length === 0} />
@@ -330,6 +331,7 @@ const CheckResultsTab = ({
{ @@ -386,12 +389,11 @@ const CheckResultsTab = ({ {results.length === 0 && (
No Data
)} - - {mode === 'table' && ( + {mode === 'table' && results.length !== 0 && ( <> {results[0] && (
({ ...item, @@ -401,13 +403,13 @@ const CheckResultsTab = ({ ).format('YYYY-MM-DD HH:mm:ss'), timePeriod: item.timePeriod?.replace(/T/g, ' ') }))} - emptyMessage="No Data" + emptyMessage="No data" getRowClass={getSeverityClass} /> )} )} - {mode === 'chart' && ( + {mode === 'chart' && results.length !== 0 && ( <> diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/ErrorSamplesTab.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/ErrorSamplesTab.tsx index 8154c5d2a1..57d0bb075a 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/ErrorSamplesTab.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/ErrorSamplesTab.tsx @@ -1,4 +1,4 @@ -import { IconButton } from '@material-tailwind/react'; +import { IconButton, Tooltip } from '@material-tailwind/react'; import moment from 'moment/moment'; import React, { useMemo } from 'react'; import { ErrorSamplesListModel } from '../../../api'; @@ -76,7 +76,7 @@ const ErrorSamplesTab = ({ { label: 'Id', value: 'id', - className: 'text-sm !py-2 whitespace-nowrap text-gray-700 text-right' + className: 'text-sm !py-2 whitespace-nowrap text-gray-700 text-left' } ]; @@ -124,31 +124,38 @@ const ErrorSamplesTab = ({ /> - - - + + + + {errorSamples.length === 0 && (
No Data
)} {errorSamples.map((result, index) => ( -
+
({ ...item, @@ -157,7 +164,7 @@ const ErrorSamplesTab = ({ getLocalDateInUserTimeZone(new Date(String(item.collectedAt))) ).format('YYYY-MM-DD HH:mm:ss') }))} - emptyMessage="No Data" + emptyMessage="No data" /> ))} diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/SensorReadoutsTab.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/SensorReadoutsTab.tsx index 8f156a3241..c1da26a41a 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/SensorReadoutsTab.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckDetails/SensorReadoutsTab.tsx @@ -41,22 +41,22 @@ const SensorReadoutsTab = ({ className: 'text-sm !py-2 whitespace-nowrap text-gray-700' }, { - label: 'Time Scale', + label: 'Time scale', value: 'timeGradient', className: 'text-sm !py-2 whitespace-nowrap text-gray-700' }, { - label: 'Time Period', + label: 'Time period', value: 'timePeriod', className: 'text-sm !py-2 whitespace-nowrap text-gray-700' }, { - label: 'Executed At', + label: 'Executed at', value: 'executedAt', className: 'text-sm !py-2 whitespace-nowrap text-gray-700' }, { - label: 'Actual Value', + label: 'Actual value', value: 'actualValue', className: 'text-sm !py-2 whitespace-nowrap text-gray-700 text-right', render: (value: number | string) => ( @@ -64,19 +64,19 @@ const SensorReadoutsTab = ({ ) }, { - label: 'Duration Ms', + label: 'Duration ms', value: 'durationMs', className: 'text-sm !py-2 whitespace-nowrap text-gray-700 text-right' }, { - label: 'Data Grouping', + label: 'Data group', value: 'dataGroup', className: 'text-sm !py-2 whitespace-nowrap text-gray-700' }, { label: 'Id', value: 'id', - className: 'text-sm !py-2 whitespace-nowrap text-gray-700 text-right' + className: 'text-sm !py-2 whitespace-nowrap text-gray-700 text-left' } ]; @@ -97,7 +97,7 @@ const SensorReadoutsTab = ({ return (
@@ -112,6 +112,7 @@ const SensorReadoutsTab = ({ })) || [] } onChange={onChangeDataGroup} + disabled={(sensorReadouts[0]?.dataGroupNames ?? []).length === 0} />
@@ -129,14 +130,15 @@ const SensorReadoutsTab = ({ {sensorReadouts.map((result, index) => (
({ ...item, executedAt: moment( getLocalDateInUserTimeZone(new Date(String(item.executedAt))) - ).format('YYYY-MM-DD HH:mm:ss') + ).format('YYYY-MM-DD HH:mm:ss'), + timePeriod: item.timePeriod?.replace(/T/g, ' ') })) || [] } /> diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckRuleItem.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckRuleItem.tsx index 6db8e1e801..44dc7bcc1c 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckRuleItem.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckRuleItem.tsx @@ -73,27 +73,34 @@ const CheckRuleItem = ({
{parameters?.configured === true ? (
{!isSimpleMode && ( { onChange({ ...parameters, diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckTableHeader.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckTableHeader.tsx index 26d4f60b1f..88c6925afc 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/CheckTableHeader.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/CheckTableHeader.tsx @@ -1,4 +1,5 @@ -import React, { useState } from 'react'; +import clsx from 'clsx'; +import React, { useEffect, useState } from 'react'; import { useSelector } from 'react-redux'; import { useHistory } from 'react-router-dom'; import { @@ -28,7 +29,6 @@ import Button from '../Button'; import Checkbox from '../Checkbox'; import DeleteOnlyDataDialog from '../CustomTree/DeleteOnlyDataDialog'; import SvgIcon from '../SvgIcon'; -import CategoryMenu from './CategoryMenu'; interface TableHeaderProps { checksUI: CheckContainerModel; @@ -44,6 +44,8 @@ interface TableHeaderProps { setShowAdvanced: (showAdvanced: boolean) => void; isFiltered?: boolean; ruleParamenterConfigured: boolean; + flashRunChecks?: boolean; + getCheckOverview?: () => void; } const TableHeader = ({ @@ -58,7 +60,9 @@ const TableHeader = ({ showAdvanced, setShowAdvanced, isFiltered, - ruleParamenterConfigured + ruleParamenterConfigured, + flashRunChecks, + getCheckOverview }: TableHeaderProps) => { const { job_dictionary_state } = useSelector( (state: IRootState) => state.job || {} @@ -82,7 +86,6 @@ const TableHeader = ({ const firstLevelActiveTab = useSelector(getFirstLevelActiveTab(checkTypes)); const { currentJobId } = useSelector(getFirstLevelState(checkTypes)); const job = currentJobId ? job_dictionary_state[currentJobId] : undefined; - const onRunChecks = async () => { await onUpdate(); const res = await JobApiClient.runChecks(undefined, false, undefined, { @@ -277,8 +280,18 @@ const TableHeader = ({ history.push(url); }; + useEffect(() => { + if (!getCheckOverview) return; + if ( + job?.status === DqoJobHistoryEntryModelStatusEnum.finished || + job?.status === DqoJobHistoryEntryModelStatusEnum.failed + ) { + getCheckOverview(); + } + }, [job?.status]); + return ( -
+ {ruleParamenterConfigured && ( { + dispatch( + setCurrentJobId( + checkTypes, + firstLevelActiveTab, + res.data?.jobId?.jobId ?? 0 + ) + ); }); }} /> diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/FieldControl.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/FieldControl.tsx index b6504e7a13..9d80ec014f 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/FieldControl.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/FieldControl.tsx @@ -6,6 +6,7 @@ import { ParameterDefinitionSpecDisplayHintEnum } from '../../api'; import CheckboxColumn from '../Checkbox/CheckBoxColumn'; +import ColumnsRecordDialog from '../ColumnsRecordDialog/ColumnsRecordDialog'; import ExtendedTextAre from '../ExtendedTextArea'; import FieldDatePicker from '../FieldDatePicker'; import FloatingPointInput from '../FloatingPointInput'; @@ -188,42 +189,57 @@ const FieldControl = ({ )} {field?.definition?.data_type === ParameterDefinitionSpecDataTypeEnum.enum && ( - - handleChange({ string_list_value: value }) + value={value} + options={ + field.definition?.allowed_values?.map((item) => ({ + label: item, + value: item + })) || [] } - onSave={onSave} + tooltipText={tooltip} + triggerClassName={clsx( + '!h-8 !text-xs !min-w-30 !max-w-30 !bg-white !ml-2', + className ? className : '!min-w-30 !max-w-30' + )} + menuClassName="!top-13" + className="text-sm" + onChange={(value) => handleChange({ enum_value: value })} disabled={disabled} + error={isInvalid} /> )} + {field?.definition?.data_type === + ParameterDefinitionSpecDataTypeEnum.string_list && ( +
+ {field.definition?.display_hint === 'column_names' ? ( + + handleChange({ string_list_value: value }) + } + onSave={onSave} + disabled={disabled} + /> + ) : ( + + handleChange({ string_list_value: value }) + } + onSave={onSave} + disabled={disabled} + /> + )} +
+ )} {field?.definition?.data_type === ParameterDefinitionSpecDataTypeEnum.integer_list && (
diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/QualityDimension/QualityDimensionStatuses.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/QualityDimension/QualityDimensionStatuses.tsx new file mode 100644 index 0000000000..17ba4509bb --- /dev/null +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/QualityDimension/QualityDimensionStatuses.tsx @@ -0,0 +1,92 @@ +import { Tooltip } from '@material-tailwind/react'; +import clsx from 'clsx'; +import React from 'react'; +import { DimensionCurrentDataQualityStatusModelCurrentSeverityEnum } from '../../../api'; +import { getDimensionColor } from '../../Connection/TableView/TableQualityStatus/TableQualityStatusUtils'; +import QualityDimensionTooltip from './QualityDimensionTooltip'; + +export default function QualityDimensionStatuses({ + dimensions +}: { + dimensions: any; +}) { + const getBasicDimmensionsKeys = (type: string) => { + const basicDimensions = Object.keys(dimensions ?? {})?.find( + (x) => x === type + ); + return basicDimensions; + }; + const basicDimensionTypes = ['Completeness', 'Validity', 'Consistency']; + + const getAdditionalDimentionsKeys = () => { + return ( + Object.keys(dimensions ?? {})?.filter( + (x) => !basicDimensionTypes.includes(x) + ) ?? [] + ); + }; + + return ( +
+ {basicDimensionTypes.map((dimType) => { + const dimensionKey = getBasicDimmensionsKeys(dimType); + const currentSeverity = + dimensions?.[dimensionKey as any]?.current_severity; + const lastCheckExecutedAt = + dimensions?.[dimensionKey as any]?.last_check_executed_at; + const severityColor = getDimensionColor(currentSeverity as any); + const hasNoSeverity = severityColor.length === 0; + + const dimensionsClassNames = clsx('w-3 h-3 border border-gray-150', { + 'bg-gray-150': hasNoSeverity && lastCheckExecutedAt, + [severityColor]: !hasNoSeverity + }); + return ( + + } + > +
+ + ); + })} + {getAdditionalDimentionsKeys().map((dimensionKey: string, dimIndex) => { + return ( + + } + > +
+ + ); + })} +
+ ); +} diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/QualityDimension/QualityDimensionTooltip.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/QualityDimension/QualityDimensionTooltip.tsx new file mode 100644 index 0000000000..cd5b061f8f --- /dev/null +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/QualityDimension/QualityDimensionTooltip.tsx @@ -0,0 +1,73 @@ +import moment from 'moment'; +import React from 'react'; +import { DimensionCurrentDataQualityStatusModel } from '../../../api'; + +const QualityDimensionTooltip = ({ + data +}: { + data: DimensionCurrentDataQualityStatusModel | undefined; +}) => { + if (data && data.last_check_executed_at) { + return ( +
+
+
Last executed at:
+
+ {moment(data?.last_check_executed_at).format('YYYY-MM-DD HH:mm:ss')} +
+
+
+
Current severity level:
+
{data?.current_severity}
+
+
+
Highest historical severity level:
+
{data?.highest_historical_severity}
+
+
+
Quality Dimension:
+
{data?.dimension}
+
+
+
Executed checks:
+
{data?.executed_checks}
+
+
+
Valid results:
+
{data?.valid_results}
+
+
+
Warnings:
+
{data?.warnings}
+
+
+
Errors:
+
{data?.errors}
+
+
+
Fatals:
+
{data?.fatals}
+
+
+
Data quality KPI:
+
+ {data.data_quality_kpi !== undefined + ? Number(data.data_quality_kpi).toFixed(2) + ' %' + : '-'} +
+
+
+ ); + } + return ( +
+
+
Quality Dimension:
+
{data?.dimension}
+
+
No data quality checks configured
+
+ ); +}; + +export default QualityDimensionTooltip; diff --git a/dqops/src/main/frontend/src/components/DataQualityChecks/index.tsx b/dqops/src/main/frontend/src/components/DataQualityChecks/index.tsx index d7fb0c682d..8a6834080c 100644 --- a/dqops/src/main/frontend/src/components/DataQualityChecks/index.tsx +++ b/dqops/src/main/frontend/src/components/DataQualityChecks/index.tsx @@ -25,7 +25,11 @@ import { } from '../../redux/selectors'; import { RUN_CHECK_TIME_WINDOW_FILTERS } from '../../shared/constants'; import { CheckTypes, ROUTES } from '../../shared/routes'; -import { useDecodedParams } from '../../utils'; +import { + getIsAnyCheckResults, + getIsAnyChecksEnabledOrDefault, + useDecodedParams +} from '../../utils'; import Button from '../Button'; import Loader from '../Loader'; import Select from '../Select'; @@ -554,14 +558,16 @@ const DataQualityChecks = ({
The results are date partitioned (grouped) by a column: - {checksUI.partition_by_column ? ( - checksUI.partition_by_column - ) : '' - } + {checksUI.partition_by_column ? checksUI.partition_by_column : ''}
{!checksUI.partition_by_column ? ( -
Warning: Partition checks will not be run, please configure the date or datetime column
- ) : ''} +
+ Warning: Partition checks will not be run, please configure the + date or datetime column +
+ ) : ( + '' + )}
{(checksUI?.categories ?? []).map((category, index) => ( diff --git a/dqops/src/main/frontend/src/components/DefinitionLayout/DefinitionTree.tsx b/dqops/src/main/frontend/src/components/DefinitionLayout/DefinitionTree.tsx index 5f756bbd86..67dd13b72c 100644 --- a/dqops/src/main/frontend/src/components/DefinitionLayout/DefinitionTree.tsx +++ b/dqops/src/main/frontend/src/components/DefinitionLayout/DefinitionTree.tsx @@ -26,8 +26,8 @@ import RuleContextMenu from './RuleContextMenu'; import SensorContextMenu from './SensorContextMenu'; const defaultChecks = [ - 'Table-level checks patterns', - 'Column-level checks patterns' + 'Table-level quality policies', + 'Column-level quality policies' ]; export const DefinitionTree = () => { @@ -74,12 +74,15 @@ export const DefinitionTree = () => { }, [refreshChecksTreeIndicator]); useEffect(() => { - toggleTree(tabs); + toggleTree(tabs, activeTab, definitionFirstLevelFolder); }, [activeTab]); - const highlightedNode = urlencodeEncoder( - activeTab?.split('/').at(activeTab?.split('/').length - 1) - ); + const highlightedNode = + activeTab && typeof activeTab === 'string' + ? urlencodeEncoder( + activeTab?.split('/')?.at(activeTab?.split('/')?.length - 1) + ) + : false; const renderSensorFolderTree = ( folder?: SensorFolderModel, @@ -421,7 +424,9 @@ export const DefinitionTree = () => { onClick={onClick} className={clsx( 'cursor-pointer flex space-x-1 items-center mb-1 h-4.5 hover:bg-gray-300', - highlightedNode === text.toLowerCase().replace(' ', '-') && + (highlightedNode === text.toLowerCase().replace(' ', '-') || + (highlightedNode === 'default-notifications' && + text === 'Global incident notifications')) && 'bg-gray-300' )} > @@ -433,7 +438,7 @@ export const DefinitionTree = () => { ); return ( -
+
{definitionFirstLevelFolder?.map((x, index) => (
{ {renderChecksFolderTree(checksFolderTree, [])}
)} - {x.category === 'Default checks configuration' && - x.isOpen === true && ( -
- {defaultChecks.map((x, index) => ( -
-
{ - openDefaultChecksPatternsFirstLevelTab( - x, - x.includes('Column') ? 'column' : 'table' - ); - }} - > - -
- {x} -
+ {x.category === 'Data quality policies' && x.isOpen === true && ( +
+ {defaultChecks.map((x, index) => ( +
+
{ + openDefaultChecksPatternsFirstLevelTab( + x, + x.includes('Column') ? 'column' : 'table' + ); + }} + > + +
+ {x}
- ))} -
- )} +
+ ))} +
+ )}
))} {(nodes as any[]).map((tab, index) => ( diff --git a/dqops/src/main/frontend/src/components/DefinitionLayout/index.tsx b/dqops/src/main/frontend/src/components/DefinitionLayout/index.tsx index e6c1643b6f..5a0d22300c 100644 --- a/dqops/src/main/frontend/src/components/DefinitionLayout/index.tsx +++ b/dqops/src/main/frontend/src/components/DefinitionLayout/index.tsx @@ -5,7 +5,6 @@ import { useHistory, useLocation } from 'react-router-dom'; import { useDefinition } from '../../contexts/definitionContext'; import CheckDetail from '../../pages/CheckDetail'; import DataDictionary from '../../pages/DataDictionaryConfiguration'; -import DataDictionaryItemOverview from '../../pages/DataDictionaryConfiguration/DataDictionaryItemOverview'; import DefaultCheckPatternConfiguration from '../../pages/DefaultCheckPatternConfiguration'; import DefaultCheckPatterns from '../../pages/DefaultCheckPatterns/DefaultCheckPatterns'; import DefaultSchedules from '../../pages/DefaultSchedulesDetail'; @@ -15,7 +14,6 @@ import IncidentConnection from '../../pages/IncidentConnection'; import RuleDetail from '../../pages/RuleDetail'; import SensorDetail from '../../pages/SensorDetail'; import SharedCredentialsDetail from '../../pages/SharedCredentialsDetail'; -import SingleSharedCredential from '../../pages/SharedCredentialsDetail/SingleSharedCredential'; import UserListDetail from '../../pages/UserListDetail'; import UserDetail from '../../pages/UserListDetail/UserDetail'; import { @@ -93,12 +91,12 @@ const DefinitionLayout = ({ route }: LayoutProps) => { return ; case ROUTES.PATTERNS.SHARED_CREDENTIALS_LIST_DETAIL: return ; - case ROUTES.PATTERNS.SHARED_CREDENTIALS_DETAIL: - return ; + // case ROUTES.PATTERNS.SHARED_CREDENTIALS_DETAIL: + // return ; case ROUTES.PATTERNS.DATA_DICTIONARY_LIST_DETAIL: return ; - case ROUTES.PATTERNS.DATA_DICTIONARY_DETAIL: - return ; + // case ROUTES.PATTERNS.DATA_DICTIONARY_DETAIL: + // return ; case ROUTES.PATTERNS.DEFAULT_CHECKS_PATTERNS: return ; case ROUTES.PATTERNS.DEFAULT_CHECK_PATTERN_DETAIL: diff --git a/dqops/src/main/frontend/src/components/FileFormatConfiguration/FileFormatConfiguration.tsx b/dqops/src/main/frontend/src/components/FileFormatConfiguration/FileFormatConfiguration.tsx index 08b4e7c4eb..5ce60822f4 100644 --- a/dqops/src/main/frontend/src/components/FileFormatConfiguration/FileFormatConfiguration.tsx +++ b/dqops/src/main/frontend/src/components/FileFormatConfiguration/FileFormatConfiguration.tsx @@ -120,7 +120,7 @@ export default function FileFormatConfiguration({ return null; } }; - + console.log(fileFormatType); return (
{' '} onDeletePath(index)} @@ -58,6 +59,7 @@ export default function FilePath({
- - - - ))} + ).map(([key, value], index) => { + return ( + key !== 'checkHierarchyIdsModels' && ( + + + + + ) + ); + })} {job?.parameters?.importSchemaParameters && ( <> diff --git a/dqops/src/main/frontend/src/components/Pagination/ClientSidePagination.tsx b/dqops/src/main/frontend/src/components/Pagination/ClientSidePagination.tsx new file mode 100644 index 0000000000..81640dd3df --- /dev/null +++ b/dqops/src/main/frontend/src/components/Pagination/ClientSidePagination.tsx @@ -0,0 +1,55 @@ +import clsx from 'clsx'; +import React, { useEffect, useMemo } from 'react'; +import { Pagination } from '.'; + +interface IClientSidePaginationProps { + items: T[]; + onChangeItems: (items: T[]) => void; + className?: string; +} + +export default function ClientSidePagination({ + items, + onChangeItems, + className +}: IClientSidePaginationProps) { + const [filters, setFilters] = React.useState({ + page: 1, + pageSize: 25 + }); + + const paginatedItems = useMemo(() => { + const { page, pageSize } = filters; + return items.slice((page - 1) * pageSize, page * pageSize); + }, [items, filters]); + + useEffect(() => { + onChangeItems(paginatedItems); + }, [paginatedItems, onChangeItems]); + + const onChangeFilters = (newFilters: Partial) => { + setFilters((prevFilters) => ({ + ...prevFilters, + ...newFilters + })); + }; + + const totalPages = Math.ceil(items.length / filters.pageSize); + + return ( +
+ = totalPages} + totalPages={totalPages} + onChange={(page, pageSize) => { + onChangeFilters({ + page, + pageSize + }); + }} + /> +
+ ); +} diff --git a/dqops/src/main/frontend/src/components/Pagination/index.tsx b/dqops/src/main/frontend/src/components/Pagination/index.tsx index 2f4d80fdc9..00b8818f42 100644 --- a/dqops/src/main/frontend/src/components/Pagination/index.tsx +++ b/dqops/src/main/frontend/src/components/Pagination/index.tsx @@ -9,8 +9,8 @@ const pageSizeOptions = [ value: 10 }, { - label: 20, - value: 20 + label: 25, + value: 25 }, { label: 50, @@ -64,6 +64,7 @@ export const Pagination = ({
{ return (
{}} >
{ + const proposeChecks = async (flash?: boolean) => { const addPrefix = (key: string) => { if (key.includes('*') || key.length === 0) { return key; @@ -173,11 +173,7 @@ export default function RuleMining({ const getShouldUserCollectStatisitcs = ( checksUI: CheckMiningProposalModel ) => { - if ( - checksUI && - checksUI.table_checks?.categories?.length === 0 && - Object.keys(checksUI?.column_checks ?? {}).length === 0 - ) { + if (checksUI.missing_current_statistics) { if (checkTypes === CheckTypes.PROFILING) { dispatch( setJobAllert({ @@ -197,6 +193,17 @@ export default function RuleMining({ }) ); } + } else { + if (flash && typeof flash === 'boolean') { + dispatch( + setJobAllert({ + activeTab: firstLevelActiveTab, + action: 'table-quality-status', + tooltipMessage: + 'When the "run checks" job finishes, the table quality status will show the status of all passed and failed data quality checks.' + }) + ); + } } }; @@ -204,7 +211,8 @@ export default function RuleMining({ ...configuration, category_filter: addPrefix(configuration.category_filter ?? ''), column_name_filter: addPrefix(configuration.column_name_filter ?? ''), - check_name_filter: addPrefix(configuration.check_name_filter ?? '') + check_name_filter: addPrefix(configuration.check_name_filter ?? ''), + propose_checks_from_statistics: checkTypes === CheckTypes.PROFILING }; setLoading(true); switch (checkTypes) { @@ -318,6 +326,16 @@ export default function RuleMining({ />
)} + {checkTypes !== CheckTypes.PROFILING && ( +
+ The number of propositions shown depends on the activated checks in + the Profiling section. +
+ To increase the number of propositions, you can either activate more + profiling checks manually or use the data quality rule miner in the Profiling + section. +
+ )}
{checksUI && @@ -352,13 +375,13 @@ export default function RuleMining({ proposeChecks(); setRunChecksDialogOpened(false); }} - message="Do you want to run the checks?" + message="The proposed configuration has been applied. Do you want to run activated checks?" onConfirm={() => { setRunChecksDialogOpened(false); runPartitionedChecks({ check_search_filters: runChecksJobTemplate }).then(() => { - proposeChecks(); + proposeChecks(true); }); toggleOpen(); }} diff --git a/dqops/src/main/frontend/src/components/RuleMining/RuleMiningFilters.tsx b/dqops/src/main/frontend/src/components/RuleMining/RuleMiningFilters.tsx index 2fa9fe7029..bec352757a 100644 --- a/dqops/src/main/frontend/src/components/RuleMining/RuleMiningFilters.tsx +++ b/dqops/src/main/frontend/src/components/RuleMining/RuleMiningFilters.tsx @@ -1,12 +1,14 @@ -import React from 'react'; +import React, { useState } from 'react'; import { CheckMiningParametersModel, CheckMiningParametersModelSeverityLevelEnum } from '../../api'; import Button from '../Button'; import Checkbox from '../Checkbox'; +import SectionWrapper from '../Dashboard/SectionWrapper'; import Input from '../Input'; import Select from '../Select'; +import SvgIcon from '../SvgIcon'; export default function RuleMiningFilters({ configuration, @@ -14,7 +16,8 @@ export default function RuleMiningFilters({ proposeChecks, applyChecks, isUpdated, - isUpdatedFilters + isUpdatedFilters, + isApplyDisabled }: { configuration: CheckMiningParametersModel; onChangeConfiguration: ( @@ -24,305 +27,347 @@ export default function RuleMiningFilters({ applyChecks: () => Promise; isUpdated: boolean; isUpdatedFilters: boolean; + isApplyDisabled: boolean; }) { + const [advancedParametersOpen, setAdvancedParametersOpen] = useState(false); + return ( -
+
- - onChangeConfiguration({ category_filter: e.target.value }) - } - /> - - onChangeConfiguration({ check_name_filter: e.target.value }) - } - /> - - onChangeConfiguration({ column_name_filter: e.target.value }) - } - /> - - onChangeConfiguration({ - fail_checks_at_percent_error_rows: - isNaN(Number(e.target.value)) || e.target.value === '' - ? undefined - : Number(e.target.value) - }) - } - /> -
-
-
- +
+ + onChangeConfiguration({ category_filter: e.target.value }) + } + /> + + onChangeConfiguration({ check_name_filter: e.target.value }) + } + /> + + onChangeConfiguration({ column_name_filter: e.target.value }) + } + /> +
+ +
+ - onChangeConfiguration({ copy_failed_profiling_checks: e }) + onChangeConfiguration({ + fail_checks_at_percent_error_rows: + isNaN(Number(e.target.value)) || e.target.value === '' + ? undefined + : Number(e.target.value) + }) } /> - - onChangeConfiguration({ copy_disabled_profiling_checks: e }) - } - /> - - onChangeConfiguration({ copy_profiling_checks: e }) - } - /> - - onChangeConfiguration({ propose_default_checks: e }) - } +
diff --git a/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryConfigurationTable.tsx b/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryConfigurationTable.tsx index f4898cb54e..1058e6e673 100644 --- a/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryConfigurationTable.tsx +++ b/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryConfigurationTable.tsx @@ -1,23 +1,34 @@ +import { IconButton } from '@material-tailwind/react'; import React, { useEffect, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { DataDictionaryListModel } from '../../api'; -import Button from '../../components/Button'; import ConfirmDialog from '../../components/CustomTree/ConfirmDialog'; +import ClientSidePagination from '../../components/Pagination/ClientSidePagination'; import SvgIcon from '../../components/SvgIcon'; -import { addFirstLevelTab } from '../../redux/actions/definition.actions'; +import { updateTabLabel } from '../../redux/actions/definition.actions'; import { IRootState } from '../../redux/reducers'; import { DataDictionaryApiClient } from '../../services/apiClient'; -import { ROUTES } from '../../shared/routes'; +import Loader from '../../components/Loader'; -export default function DataDictionaryConfigurationTable() { +export default function DataDictionaryConfigurationTable({ + setDictionaryToEdit +}: { + setDictionaryToEdit: (value: string) => void; +}) { const { userProfile } = useSelector((state: IRootState) => state.job || {}); const [loading, setLoading] = useState(false); const [dictionaries, setDictionaries] = useState( [] ); + const [displayedDictionaries, setDisplayedDictionaries] = useState< + DataDictionaryListModel[] + >([]); const [selectedDictionaryToDelete, setSelectedDictionaryToDelete] = useState(''); + + const { activeTab } = useSelector((state: IRootState) => state.definition); + const dispatch = useDispatch(); const getAllDictionaries = async () => { @@ -29,16 +40,8 @@ export default function DataDictionaryConfigurationTable() { }; const updateDataDictionary = async (dictionary_name: string) => { - dispatch( - addFirstLevelTab({ - url: ROUTES.DATA_DICTIONARY_DETAIL(dictionary_name), - value: ROUTES.DATA_DICTIONARY_VALUE(dictionary_name), - label: 'Edit dictionary', - state: { - dictionary_name - } - }) - ); + dispatch(updateTabLabel(dictionary_name, activeTab ?? '')); + setDictionaryToEdit(dictionary_name); }; const deleteDictionary = async (dictionary: string) => { @@ -56,66 +59,90 @@ export default function DataDictionaryConfigurationTable() { if (loading) { return (
- +
); } return ( - - {dictionaries?.map((dictionary, index) => ( - - - - - - - - ))} + <> +
Data quality check - {(!job || - job?.status === DqoJobHistoryEntryModelStatusEnum.finished || - job?.status === DqoJobHistoryEntryModelStatusEnum.failed) && - isDefaultEditing !== true && ( - setDeleteDataDialogOpened(true)} - /> - )} - {job?.status === DqoJobHistoryEntryModelStatusEnum.waiting && ( - - )} - {(job?.status === DqoJobHistoryEntryModelStatusEnum.running || - job?.status === DqoJobHistoryEntryModelStatusEnum.queued) && ( - - )}
{isFiltered !== true ? ( @@ -339,26 +330,6 @@ const TableHeader = ({
- {/* {!mode && ( - <> -
{!(index === 0 && properties.length === 1) && ( { const collectStatistics = async () => { setIsCollected(true); - JobApiClient.collectStatisticsOnTable(undefined, false, undefined, { - connection: advisorObject.connectionName - }); + if (advisorObject.tableNames && advisorObject.tableNames.length > 0) { + advisorObject.tableNames.forEach((tableName) => { + JobApiClient.collectStatisticsOnTable(undefined, true, undefined, false, undefined, { + connection: advisorObject.connectionName, + fullTableName: advisorObject.schemaName + "." + tableName + }); + }); + } else { + JobApiClient.collectStatisticsOnTable(undefined, true, undefined, false, undefined, { + connection: advisorObject.connectionName, + fullTableName: advisorObject.schemaName + ".*" + }); + } }; const runProfilingChecks = async () => { - JobApiClient.runChecks(undefined, false, undefined, { - check_search_filters: { - connection: advisorObject.connectionName, - checkType: CheckTypes.PROFILING - } - }); + if (advisorObject.tableNames && advisorObject.tableNames.length > 0) { + advisorObject.tableNames.forEach((tableName) => { + JobApiClient.runChecks(undefined, false, undefined, { + check_search_filters: { + connection: advisorObject.connectionName, + fullTableName: advisorObject.schemaName + "." + tableName, + checkType: CheckTypes.PROFILING + } + }); + }); + } else { + JobApiClient.runChecks(undefined, false, undefined, { + check_search_filters: { + connection: advisorObject.connectionName, + fullTableName: advisorObject.schemaName + ".*", + checkType: CheckTypes.PROFILING + } + }); + } }; const handleSubmit = async () => { @@ -55,7 +78,7 @@ export const HeaderBanner = ({ onClose }: HeaderBannerProps) => {
- New tables were imported into DQOps. You should collect statistics to + New tables have been imported into DQOps. It is recommended to collect statistics to enable data quality rule mining based on data samples.
@@ -80,9 +103,9 @@ export const HeaderBanner = ({ onClose }: HeaderBannerProps) => {
- Warning: If you imported a lot of tables, do not profile all tables - at once. Click the {'"'}Cancel{'"'} button and collect statistics on - each table individually + Warning: If you have imported many tables, avoid profiling all tables simultaneously + Instead, click the {'"'}Cancel{'"'} button and collect statistics for + each table individually.
diff --git a/dqops/src/main/frontend/src/components/Header/index.tsx b/dqops/src/main/frontend/src/components/Header/index.tsx index 2f74d254be..775bc073b0 100644 --- a/dqops/src/main/frontend/src/components/Header/index.tsx +++ b/dqops/src/main/frontend/src/components/Header/index.tsx @@ -80,7 +80,7 @@ const Header = () => { newCheckTypes, connection, schema, - 'tables' + 'data-quality-summary' ); value = ROUTES.SCHEMA_LEVEL_VALUE(newCheckTypes, connection, schema); } else if (match.path === ROUTES.PATTERNS.TABLE) { @@ -196,7 +196,7 @@ const Header = () => { }, []); return ( -
+
{isAdvisorOpen && (
diff --git a/dqops/src/main/frontend/src/components/HelpMenu/index.tsx b/dqops/src/main/frontend/src/components/HelpMenu/index.tsx index 9f9f9b76b8..f695a3f254 100644 --- a/dqops/src/main/frontend/src/components/HelpMenu/index.tsx +++ b/dqops/src/main/frontend/src/components/HelpMenu/index.tsx @@ -28,7 +28,7 @@ const HelpMenu = () => {
- +
(
@@ -98,6 +100,7 @@ const Input = ({ data-testid={dataTestId} defaultValue={defaultValue} onKeyDown={onKeyDown} + autoFocus={autoFocus} // Apply autoFocus prop /> {value !== undefined && (
diff --git a/dqops/src/main/frontend/src/components/Logo/index.tsx b/dqops/src/main/frontend/src/components/Logo/index.tsx index 111e9bc70a..e4c829a78e 100644 --- a/dqops/src/main/frontend/src/components/Logo/index.tsx +++ b/dqops/src/main/frontend/src/components/Logo/index.tsx @@ -8,7 +8,7 @@ interface ILogoProps { } const Logo = (props: ILogoProps) => { - return ; + return ; }; export default Logo; diff --git a/dqops/src/main/frontend/src/components/NotificationMenu/JobItem/index.tsx b/dqops/src/main/frontend/src/components/NotificationMenu/JobItem/index.tsx index b7796e01c6..f078023064 100644 --- a/dqops/src/main/frontend/src/components/NotificationMenu/JobItem/index.tsx +++ b/dqops/src/main/frontend/src/components/NotificationMenu/JobItem/index.tsx @@ -126,17 +126,21 @@ const JobItem = ({ {job?.parameters?.runChecksParameters?.check_search_filters && Object.entries( job?.parameters?.runChecksParameters?.check_search_filters - ).map(([key, value], index) => ( -
- {key - .replace(/([a-z])([A-Z])/g, '$1 $2') - .toLowerCase() - .replace(/^\w/, (c) => c.toUpperCase())} - {renderValue(value)}
+ {key + .replace(/([a-z])([A-Z])/g, '$1 $2') + .toLowerCase() + .replace(/^\w/, (c) => c.toUpperCase())} + {renderValue(value)}
onDelete(key)} @@ -96,6 +97,7 @@ const RuleParameters = ({ @@ -52,19 +52,16 @@ export const SensorActionGroup = ({ color="primary" variant="outlined" label="Copy" - className="w-40 !h-10" + className="w-40 !h-[37px]" disabled={userProfile.can_manage_definitions !== true} onClick={onCopy} />
Dictionary file name
- updateDataDictionary(dictionary.dictionary_name ?? '') - } - > - {dictionary.dictionary_name} - - {'${dictionary://' + dictionary.dictionary_name + '}'} - - - - - Download - -
+ + {displayedDictionaries?.map((dictionary, index) => ( + + + + + + + + ))} + +
+ updateDataDictionary(dictionary.dictionary_name ?? '') + } + > + {dictionary.dictionary_name} + + {'${dictionary://' + dictionary.dictionary_name + '}'} + + + updateDataDictionary(dictionary.dictionary_name ?? '') + } + ripple={false} + color="teal" + className="!shadow-none hover:!shadow-none hover:bg-[#028770]" + disabled={userProfile.can_manage_definitions !== true} + > + + + + + setSelectedDictionaryToDelete( + dictionary.dictionary_name ?? '' + ) + } + ripple={false} + disabled={userProfile.can_manage_definitions !== true} + color="teal" + className="!shadow-none hover:!shadow-none hover:bg-[#028770]" + > + + + + + + + + +
+ setSelectedDictionaryToDelete('')} onConfirm={() => deleteDictionary(selectedDictionaryToDelete)} message={`Are you sure you want to delete ${selectedDictionaryToDelete} dictionary?`} /> - + ); } diff --git a/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryItemOverview.tsx b/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryItemOverview.tsx index 4df6ec82f1..41055a1472 100644 --- a/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryItemOverview.tsx +++ b/dqops/src/main/frontend/src/pages/DataDictionaryConfiguration/DataDictionaryItemOverview.tsx @@ -1,21 +1,24 @@ import React, { useEffect, useState } from 'react'; -import { useDispatch, useSelector } from 'react-redux'; +import { useSelector } from 'react-redux'; import Button from '../../components/Button'; import Input from '../../components/Input'; +import Loader from '../../components/Loader'; +import SvgIcon from '../../components/SvgIcon'; import TextArea from '../../components/TextArea'; -import { closeFirstLevelTab } from '../../redux/actions/definition.actions'; import { IRootState } from '../../redux/reducers'; -import { getFirstLevelSensorState } from '../../redux/selectors'; import { DataDictionaryApiClient } from '../../services/apiClient'; -import { urlencodeDecoder } from '../../utils'; -export default function DataDictionaryItemOverview() { +export default function DataDictionaryItemOverview({ + dictionary_name = '', + onBack +}: { + dictionary_name?: string; + onBack: () => void; +}) { const { userProfile } = useSelector((state: IRootState) => state.job || {}); - const { dictionary_name } = useSelector(getFirstLevelSensorState); const [dictionaryName, setDictionaryName] = useState(''); const [textAreaValue, setTextAreaValue] = useState(''); - - const dispatch = useDispatch(); + const [loading, setLoading] = useState(false); const addDictionary = async () => { if (dictionaryName.length === 0) return; @@ -23,7 +26,7 @@ export default function DataDictionaryItemOverview() { dictionary_name: dictionaryName, file_content: textAreaValue }).catch((err) => console.error(err)); - dispatch(closeFirstLevelTab('/definitions/data-dictionary/new')); + onBack(); }; const editDictionary = async () => { @@ -31,26 +34,32 @@ export default function DataDictionaryItemOverview() { dictionary_name: dictionaryName, file_content: textAreaValue }).catch((err) => console.error(err)); - - dispatch( - closeFirstLevelTab( - '/definitions/data-dictionary/' + urlencodeDecoder(dictionary_name) - ) - ); + onBack(); }; useEffect(() => { - if (dictionary_name) { + if (dictionary_name && dictionary_name.length !== 0) { const getDictionary = () => { - DataDictionaryApiClient.getDictionary(dictionary_name).then((res) => { - setDictionaryName(res?.data?.dictionary_name ?? ''), - setTextAreaValue(res?.data?.file_content ?? ''); - }); + setLoading(true); + DataDictionaryApiClient.getDictionary(dictionary_name) + .then((res) => { + setDictionaryName(res?.data?.dictionary_name ?? ''), + setTextAreaValue(res?.data?.file_content ?? ''); + }) + .finally(() => setLoading(false)); }; getDictionary(); } }, [dictionary_name]); + if (loading) { + return ( +
+ +
+ ); + } + return ( <>
@@ -60,11 +69,20 @@ export default function DataDictionaryItemOverview() { value={dictionaryName} onChange={(e) => setDictionaryName(e.target.value)} disabled={ - dictionary_name || userProfile.can_manage_definitions !== true + dictionary_name.length !== 0 || + userProfile.can_manage_definitions !== true } />
+
- Pattern name + Quality policy name {create === true ? ( onChangeTarget({ pattern_name: e.target.value })} + value={target?.policy_name} + onChange={(e) => onChangeTarget({ policy_name: e.target.value })} /> ) : ( - {target?.pattern_name} + {target?.policy_name} )}
diff --git a/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/EditCheckPattern.tsx b/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/EditCheckPattern.tsx index e638dbbd6c..ff204bb6ab 100644 --- a/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/EditCheckPattern.tsx +++ b/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/EditCheckPattern.tsx @@ -3,8 +3,8 @@ import React, { useEffect, useState } from 'react'; import { useSelector } from 'react-redux'; import { CheckContainerModel, - DefaultColumnChecksPatternListModel, - DefaultTableChecksPatternListModel, + ColumnQualityPolicyListModel, + TableQualityPolicyListModel, TargetColumnPatternSpec, TargetTablePatternSpec } from '../../api'; @@ -12,10 +12,12 @@ import Button from '../../components/Button'; import DataQualityChecks from '../../components/DataQualityChecks'; import SvgIcon from '../../components/SvgIcon'; import Tabs from '../../components/Tabs'; +import { useActionDispatch } from '../../hooks/useActionDispatch'; +import { setSecondLevelTab } from '../../redux/actions/definition.actions'; import { IRootState } from '../../redux/reducers'; import { - DefaultColumnCheckPatternsApiClient, - DefaultTableCheckPatternsApiClient + ColumnQualityPoliciesApiClient, + TableQualityPoliciesApiClient } from '../../services/apiClient'; import { CheckRunMonitoringScheduleGroup } from '../../shared/enums/scheduling.enum'; import CopyCheckPatternDialog from './CopyCheckPatternDialog'; @@ -124,13 +126,11 @@ type TCheckContainerDiverse = { [key in TCheckTypes]: CheckContainerModel | undefined; }; -type TTarget = - | DefaultColumnChecksPatternListModel - | DefaultTableChecksPatternListModel; +type TTarget = ColumnQualityPolicyListModel | TableQualityPolicyListModel; type TTargetSpec = TargetColumnPatternSpec | TargetTablePatternSpec; type TEditCheckPatternProps = { type: 'table' | 'column'; - pattern_name: string; + policy_name: string; create: boolean; }; const initialState: TCheckContainerDiverse = { @@ -143,10 +143,16 @@ const initialState: TCheckContainerDiverse = { export default function EditCheckPattern({ type, - pattern_name, + policy_name, create }: TEditCheckPatternProps) { const { userProfile } = useSelector((state: IRootState) => state.job); + const { tabs: pageTabs, activeTab: baseTab } = useSelector( + (state: IRootState) => state.definition + ); + const activeTab = + (pageTabs.find((x) => x.url === baseTab)?.state?.secondTab as string) ?? + 'table-target'; const targetSpecKey = type === 'column' ? 'target_column' : 'target_table'; const tabs = userProfile && @@ -159,22 +165,21 @@ export default function EditCheckPattern({ : type === 'column' ? tabsColumnChecksFreeTrial : tabsTableChecksFreeTrial; - - const [activeTab, setActiveTab] = useState('table-target'); + const dispatch = useActionDispatch(); const [checkContainers, setCheckContainers] = useState(initialState); const [target, setTarget] = useState({}); const [isUpdated, setIsUpdated] = useState(false); const [copyPatternOpen, setCopyPatternOpen] = useState(false); - const onChangeTab = (tab: any) => { - setActiveTab(tab); + const onChangeTab = (tab: string) => { + dispatch(setSecondLevelTab(tab, baseTab ?? '')); }; const onChangeTarget = ( updatedTarget: Partial | Partial ) => { if ( - 'pattern_name' in updatedTarget || + 'policy_name' in updatedTarget || 'priority' in updatedTarget || 'description' in updatedTarget || 'disabled' in updatedTarget @@ -196,7 +201,7 @@ export default function EditCheckPattern({ }; const updateChecks = async () => { - if (!pattern_name) return; + if (!policy_name) return; const apiClients: { column: Record< TCheckTypes, @@ -217,27 +222,27 @@ export default function EditCheckPattern({ } = { column: { [CheckRunMonitoringScheduleGroup.profiling]: - DefaultColumnCheckPatternsApiClient.updateDefaultProfilingColumnChecksPattern, + ColumnQualityPoliciesApiClient.updateProfilingColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_daily]: - DefaultColumnCheckPatternsApiClient.updateDefaultMonitoringDailyColumnChecksPattern, + ColumnQualityPoliciesApiClient.updateMonitoringDailyColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_monthly]: - DefaultColumnCheckPatternsApiClient.updateDefaultMonitoringMonthlyColumnChecksPattern, + ColumnQualityPoliciesApiClient.updateMonitoringMonthlyColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_daily]: - DefaultColumnCheckPatternsApiClient.updateDefaultPartitionedDailyColumnChecksPattern, + ColumnQualityPoliciesApiClient.updatePartitionedDailyColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_monthly]: - DefaultColumnCheckPatternsApiClient.updateDefaultPartitionedMonthlyColumnChecksPattern + ColumnQualityPoliciesApiClient.updatePartitionedMonthlyColumnQualityPolicy }, table: { [CheckRunMonitoringScheduleGroup.profiling]: - DefaultTableCheckPatternsApiClient.updateDefaultProfilingTableChecksPattern, + TableQualityPoliciesApiClient.updateProfilingTableQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_daily]: - DefaultTableCheckPatternsApiClient.updateDefaultMonitoringDailyTableChecksPattern, + TableQualityPoliciesApiClient.updateMonitoringDailyTableQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_monthly]: - DefaultTableCheckPatternsApiClient.updateDefaultMonitoringMonthlyTableChecksPattern, + TableQualityPoliciesApiClient.updateMonitoringMonthlyTableQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_daily]: - DefaultTableCheckPatternsApiClient.updateDefaultPartitionedDailyTableChecksPattern, + TableQualityPoliciesApiClient.updatePartitionedDailyTableQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_monthly]: - DefaultTableCheckPatternsApiClient.updateDefaultPartitionedMonthlyTableChecksPattern + TableQualityPoliciesApiClient.updatePartitionedMonthlyTableQualityPolicy } }; @@ -253,7 +258,7 @@ export default function EditCheckPattern({ ) { const apiFunction = apiClient[key as TCheckTypes]; promises.push( - apiFunction(pattern_name, checkContainers[key as TCheckTypes]) + apiFunction(policy_name, checkContainers[key as TCheckTypes]) ); } }); @@ -261,13 +266,13 @@ export default function EditCheckPattern({ await Promise.all(promises); if (type === 'column') { - await DefaultColumnCheckPatternsApiClient.updateDefaultColumnChecksPatternTarget( - pattern_name, + await ColumnQualityPoliciesApiClient.updateColumnQualityPolicyTarget( + policy_name, target ); } else { - await DefaultTableCheckPatternsApiClient.updateDefaultTableChecksPatternTarget( - pattern_name, + await TableQualityPoliciesApiClient.updateTableQualityPolicyTarget( + policy_name, target ); } @@ -283,14 +288,14 @@ export default function EditCheckPattern({ }; const getTarget = () => { - if (!pattern_name) return; + if (!policy_name) return; if (type === 'column') { - DefaultColumnCheckPatternsApiClient.getDefaultColumnChecksPatternTarget( - pattern_name + ColumnQualityPoliciesApiClient.getColumnQualityPolicyTarget( + policy_name ).then((res) => setTarget(res?.data)); } else { - DefaultTableCheckPatternsApiClient.getDefaultTableChecksPatternTarget( - pattern_name + TableQualityPoliciesApiClient.getTableQualityPolicyTarget( + policy_name ).then((res) => setTarget(res?.data)); } }; @@ -305,27 +310,27 @@ export default function EditCheckPattern({ const apiClients = { column: { [CheckRunMonitoringScheduleGroup.profiling]: - DefaultColumnCheckPatternsApiClient.getDefaultProfilingColumnChecksPattern, + ColumnQualityPoliciesApiClient.getProfilingColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_daily]: - DefaultColumnCheckPatternsApiClient.getDefaultMonitoringDailyColumnChecksPattern, + ColumnQualityPoliciesApiClient.getMonitoringDailyColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_monthly]: - DefaultColumnCheckPatternsApiClient.getDefaultMonitoringMonthlyColumnChecksPattern, + ColumnQualityPoliciesApiClient.getMonitoringMonthlyColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_daily]: - DefaultColumnCheckPatternsApiClient.getDefaultPartitionedDailyColumnChecksPattern, + ColumnQualityPoliciesApiClient.getPartitionedDailyColumnQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_monthly]: - DefaultColumnCheckPatternsApiClient.getDefaultPartitionedMonthlyColumnChecksPattern + ColumnQualityPoliciesApiClient.getPartitionedMonthlyColumnQualityPolicy }, table: { [CheckRunMonitoringScheduleGroup.profiling]: - DefaultTableCheckPatternsApiClient.getDefaultProfilingTableChecksPattern, + TableQualityPoliciesApiClient.getProfilingTableQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_daily]: - DefaultTableCheckPatternsApiClient.getDefaultMonitoringDailyTableChecksPattern, + TableQualityPoliciesApiClient.getMonitoringDailyTableQualityPolicy, [CheckRunMonitoringScheduleGroup.monitoring_monthly]: - DefaultTableCheckPatternsApiClient.getDefaultMonitoringMonthlyTableChecksPattern, + TableQualityPoliciesApiClient.getMonitoringMonthlyTableQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_daily]: - DefaultTableCheckPatternsApiClient.getDefaultPartitionedDailyTableChecksPattern, + TableQualityPoliciesApiClient.getPartitionedDailyTableQualityPolicy, [CheckRunMonitoringScheduleGroup.partitioned_monthly]: - DefaultTableCheckPatternsApiClient.getDefaultPartitionedMonthlyTableChecksPattern + TableQualityPoliciesApiClient.getPartitionedMonthlyTableQualityPolicy } }; @@ -333,12 +338,13 @@ export default function EditCheckPattern({ const apiCall = apiClient[activeTab as CheckRunMonitoringScheduleGroup]; if (apiCall) { - await apiCall(pattern_name).then(callBack); + if (!policy_name) return; + await apiCall(policy_name).then(callBack); } }; useEffect(() => { - if (!pattern_name) return; + if (!policy_name) return; if ( !checkContainers?.[activeTab as TCheckTypes] || Object.keys(checkContainers?.[activeTab as TCheckTypes] ?? {}).length === @@ -349,13 +355,13 @@ export default function EditCheckPattern({ }, [activeTab]); useEffect(() => { - if (!pattern_name) return; + if (!policy_name) return; getChecks(); - }, [pattern_name, create, type]); + }, [policy_name, create, type]); useEffect(() => { getTarget(); - }, [pattern_name]); + }, [policy_name]); useEffect(() => { getTarget(); @@ -371,7 +377,7 @@ export default function EditCheckPattern({
{type?.replace(/./, (c) => c.toUpperCase())} check pattern{' '} - {pattern_name} + {policy_name}
@@ -421,7 +427,7 @@ export default function EditCheckPattern({
diff --git a/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/index.tsx b/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/index.tsx index 5a88ba5624..de953e2b2e 100644 --- a/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/index.tsx +++ b/dqops/src/main/frontend/src/pages/DefaultCheckPatternConfiguration/index.tsx @@ -8,9 +8,9 @@ import EditCheckPattern from './EditCheckPattern'; export default function index() { const { type, - pattern_name, + policy_name, create - }: { type: 'table' | 'column'; pattern_name: string; create: boolean } = + }: { type: 'table' | 'column'; policy_name: string; create: boolean } = useSelector(getFirstLevelSensorState); return ( @@ -20,7 +20,7 @@ export default function index() { ) : ( )} diff --git a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/ColumnLevelPatterns.tsx b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/ColumnLevelPatterns.tsx index 5e63f6ee7b..c370c0e243 100644 --- a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/ColumnLevelPatterns.tsx +++ b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/ColumnLevelPatterns.tsx @@ -1,19 +1,47 @@ import React, { useEffect, useState } from 'react'; -import { DefaultColumnCheckPatternsApiClient } from '../../services/apiClient'; +import { ColumnQualityPoliciesApiClient } from '../../services/apiClient'; -import DefaultCheckPatternsTable from './DefaultCheckPatternsTable'; -import { DefaultColumnChecksPatternListModel } from '../../api'; +import { ColumnQualityPolicyListModel } from '../../api'; +import Loader from '../../components/Loader'; import { sortPatterns } from '../../utils'; +import DefaultCheckPatternsTable from './DefaultCheckPatternsTable'; + +const getPreparedPatterns = ( + patterns: ColumnQualityPolicyListModel[] +) => { + const arr: any[] = []; + + patterns.forEach((x) => { + const targetSpec: any = x.target_column; + if ( + targetSpec && + typeof targetSpec === 'object' && + Object.keys(targetSpec).length !== 0 + ) { + arr.push({ ...x, ...targetSpec }); + } else { + arr.push(x); + } + }); + + return arr; +}; export default function ColumnLevelPatterns() { const [patterns, setPatterns] = useState< - DefaultColumnChecksPatternListModel[] + ColumnQualityPolicyListModel[] >([]); + const [loading, setLoading] = useState(false); const getPatterns = async () => { - DefaultColumnCheckPatternsApiClient.getAllDefaultColumnChecksPatterns().then( - (res) => setPatterns(sortPatterns(res.data ?? [], 'priority', 'asc')) - ); + setLoading(true); + ColumnQualityPoliciesApiClient.getColumnQualityPolicies() + .then((res) => + setPatterns( + sortPatterns(getPreparedPatterns(res.data ?? []), 'priority', 'asc') + ) + ) + .finally(() => setLoading(false)); }; useEffect(() => { @@ -21,11 +49,19 @@ export default function ColumnLevelPatterns() { }, []); const deletePattern = (patternName: string) => { - DefaultColumnCheckPatternsApiClient.deleteDefaultColumnChecksPattern( + ColumnQualityPoliciesApiClient.deleteColumnQualityPolicy( patternName ).then(() => getPatterns()); }; + if (loading) { + return ( +
+ +
+ ); + } + return (
diff --git a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatterns.tsx b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatterns.tsx index 8c1e608671..b7136e09c9 100644 --- a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatterns.tsx +++ b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatterns.tsx @@ -13,7 +13,7 @@ export default function DefaultCheckPatterns() { const { openDefaultCheckPatternFirstLevelTab } = useDefinition(); - const addPattern = () => { + const addQualityPolicy = () => { openDefaultCheckPatternFirstLevelTab(type, 'new pattern', { create: true }); }; @@ -22,14 +22,15 @@ export default function DefaultCheckPatterns() {
- Default {type}-level check patterns + {type?.replace(/./, (x) => x.toUpperCase())}-level data quality + policies
{type === 'table' ? : } diff --git a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatternsTable.tsx b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatternsTable.tsx index 1770839a60..e3730eebaf 100644 --- a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatternsTable.tsx +++ b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/DefaultCheckPatternsTable.tsx @@ -1,20 +1,24 @@ +import { IconButton } from '@material-tailwind/react'; import clsx from 'clsx'; import React, { useState } from 'react'; import { useSelector } from 'react-redux'; import { - DefaultColumnChecksPatternListModel, - DefaultTableChecksPatternListModel + ColumnQualityPolicyListModel, + TableQualityPolicyListModel } from '../../api'; -import Button from '../../components/Button'; import ConfirmDialog from '../../components/CustomTree/ConfirmDialog'; +import ClientSidePagination from '../../components/Pagination/ClientSidePagination'; // Import pagination component import SvgIcon from '../../components/SvgIcon'; +import Switch from '../../components/Switch'; import { useDefinition } from '../../contexts/definitionContext'; import { getFirstLevelSensorState } from '../../redux/selectors'; +import { + ColumnQualityPoliciesApiClient, + TableQualityPoliciesApiClient +} from '../../services/apiClient'; import { sortPatterns } from '../../utils'; -type TPattern = - | DefaultTableChecksPatternListModel - | DefaultColumnChecksPatternListModel; +type TPattern = TableQualityPolicyListModel | ColumnQualityPolicyListModel; type TDefaultCheckPatternsTableProps = { patterns: TPattern[]; @@ -25,31 +29,34 @@ type TDefaultCheckPatternsTableProps = { type THeaderElement = { label: string; key: - | 'pattern_name' + | 'policy_name' | 'priority' | 'can_edit' | 'yaml_parsing_error' | 'connection' | 'schema' | 'table' - | 'column'; + | 'column' + | 'action'; }; const headerElementTablePatterns: THeaderElement[] = [ - { label: 'Pattern name', key: 'pattern_name' }, + { label: 'Quality policy name', key: 'policy_name' }, { label: 'Priority', key: 'priority' }, { label: 'Connection', key: 'connection' }, { label: 'Schema', key: 'schema' }, - { label: 'Table', key: 'table' } + { label: 'Table', key: 'table' }, + { label: 'Action', key: 'action' } ]; const headerElementColumnPatterns: THeaderElement[] = [ - { label: 'Pattern name', key: 'pattern_name' }, + { label: 'Quality policy name', key: 'policy_name' }, { label: 'Priority', key: 'priority' }, { label: 'Connection', key: 'connection' }, { label: 'Schema', key: 'schema' }, { label: 'Table', key: 'table' }, - { label: 'Column', key: 'column' } + { label: 'Column', key: 'column' }, + { label: 'Action', key: 'action' } ]; export default function DefaultCheckPatternsTable({ @@ -67,120 +74,164 @@ export default function DefaultCheckPatternsTable({ const [dir, setDir] = useState<'asc' | 'desc'>('desc'); const [patternDelete, setPatternDelete] = useState(''); const [indexSortingElement, setIndexSortingElement] = useState(1); - const targetSpecKey = type === 'column' ? 'target_column' : 'target_table'; + const [displayedPatterns, setDisplayedPatterns] = useState([]); // State for displayed patterns + const headerElement = type === 'column' ? headerElementColumnPatterns : headerElementTablePatterns; - const getPreparedPatterns = () => { - const arr: any[] = []; - - patterns.forEach((x) => { - const targetSpec: any = x[targetSpecKey as keyof TPattern]; - if ( - targetSpec && - typeof targetSpec === 'object' && - Object.keys(targetSpec).length !== 0 - ) { - arr.push({ ...x, ...targetSpec }); - } else { - arr.push(x); - } - }); - - return arr; - }; - const sortPreparedPattern = ( elem: THeaderElement, index: number, dir: 'asc' | 'desc' ) => { - onChange( - sortPatterns(getPreparedPatterns(), elem.key as keyof TPattern, dir) - ), + onChange(sortPatterns(patterns, elem.key as keyof TPattern, dir)), setDir(dir === 'asc' ? 'desc' : 'asc'), setIndexSortingElement(index); }; + const handleDisablePattern = (pattern: TPattern) => { + const newPatterns = patterns.map((x) => { + if (x.policy_name === pattern.policy_name) { + x.disabled = !x.disabled; + } + return x; + }); + if (type === 'table') { + TableQualityPoliciesApiClient.getTableQualityPolicyTarget( + pattern.policy_name ?? '' + ).then((res) => { + TableQualityPoliciesApiClient.updateTableQualityPolicyTarget( + res.data.policy_name ?? '', + { + ...res.data, + disabled: !res.data.disabled + } + ); + }); + } else { + ColumnQualityPoliciesApiClient.getColumnQualityPolicyTarget( + pattern.policy_name ?? '' + ).then((res) => { + ColumnQualityPoliciesApiClient.updateColumnQualityPolicyTarget( + res.data.policy_name ?? '', + { + ...res.data, + disabled: !res.data.disabled + } + ); + }); + } + onChange(newPatterns); + }; + return ( - - - - {headerElement.map((elem, index) => ( - + + {displayedPatterns.map((pattern, index) => ( + + + + + + + + {type === 'column' && } + + + ))} + +
-
-
{elem.label}
-
- {!(indexSortingElement === index && dir === 'asc') ? ( - sortPreparedPattern(elem, index, 'desc')} - /> - ) : ( -
+
+ + + + + {headerElement.map((elem, index) => ( + - ))} - - - - {getPreparedPatterns().map((pattern, index) => ( - - - - - - - {type === 'column' && } - - + + ))} - ))} - 0} - onConfirm={async () => { - deletePattern(patternDelete ?? ''), setPatternDelete(''); - }} - onClose={() => setPatternDelete('')} - message={`Are you sure you want to delete the ${patternDelete} pattern ?`} - /> - -
+
sortPreparedPattern(elem, index, 'asc')} - /> - ) : ( -
+ > +
{elem.label}
+ {elem.key !== 'action' && ( +
+ {!(indexSortingElement === index && dir === 'asc') ? ( + + sortPreparedPattern(elem, index, 'desc') + } + /> + ) : ( +
+ )} + {!(indexSortingElement === index && dir === 'desc') ? ( + + sortPreparedPattern(elem, index, 'asc') + } + /> + ) : ( +
+ )} +
)}
-
-
editPattern(type, pattern.pattern_name ?? '')} - > - {pattern.pattern_name} - {pattern.priority}{pattern?.connection}{pattern?.schema}{pattern?.table}{pattern?.column} - -
+
+ handleDisablePattern(pattern)} + /> + editPattern(type, pattern.policy_name ?? '')} + > + {pattern.policy_name} + {pattern.priority}{pattern?.connection}{pattern?.schema}{pattern?.table}{pattern?.column} +
+ editPattern(type, pattern.policy_name ?? '')} + ripple={false} + color="teal" + className="!shadow-none hover:!shadow-none hover:bg-[#028770]" + > + + + setPatternDelete(pattern.policy_name ?? '')} + color="teal" + className="!shadow-none hover:!shadow-none hover:bg-[#028770]" + > + + +
+
+ + 0} + onConfirm={async () => { + deletePattern(patternDelete ?? ''), setPatternDelete(''); + }} + onClose={() => setPatternDelete('')} + message={`Are you sure you want to delete the ${patternDelete} pattern?`} + /> +
); } diff --git a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/TableLevelPatterns.tsx b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/TableLevelPatterns.tsx index f0cdb3cb4b..e5c5d90988 100644 --- a/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/TableLevelPatterns.tsx +++ b/dqops/src/main/frontend/src/pages/DefaultCheckPatterns/TableLevelPatterns.tsx @@ -1,29 +1,63 @@ import React, { useEffect, useState } from 'react'; -import { DefaultTableCheckPatternsApiClient } from '../../services/apiClient'; +import { TableQualityPoliciesApiClient } from '../../services/apiClient'; -import DefaultCheckPatternsTable from './DefaultCheckPatternsTable'; -import { DefaultTableChecksPatternListModel } from '../../api'; +import { TableQualityPolicyListModel } from '../../api'; +import Loader from '../../components/Loader'; import { sortPatterns } from '../../utils'; +import DefaultCheckPatternsTable from './DefaultCheckPatternsTable'; +const getPreparedPatterns = ( + patterns: TableQualityPolicyListModel[] +) => { + const arr: any[] = []; + patterns.forEach((x) => { + const targetSpec: any = x.target_table; + if ( + targetSpec && + typeof targetSpec === 'object' && + Object.keys(targetSpec).length !== 0 + ) { + arr.push({ ...x, ...targetSpec }); + } else { + arr.push(x); + } + }); + + return arr; +}; export default function TableLevelPatterns() { const [patterns, setPatterns] = useState< - DefaultTableChecksPatternListModel[] + TableQualityPolicyListModel[] >([]); + const [loading, setLoading] = useState(false); const getPatterns = async () => { - DefaultTableCheckPatternsApiClient.getAllDefaultTableChecksPatterns().then( - (res) => setPatterns(sortPatterns(res.data ?? [], 'priority', 'asc')) - ); + setLoading(true); + TableQualityPoliciesApiClient.getTableQualityPolicies() + .then((res) => { + setPatterns( + sortPatterns(getPreparedPatterns(res.data ?? []), 'priority', 'asc') + ); + }) + .finally(() => setLoading(false)); }; useEffect(() => { getPatterns(); }, []); const deletePattern = (patternName: string) => { - DefaultTableCheckPatternsApiClient.deleteDefaultTableChecksPattern( + TableQualityPoliciesApiClient.deleteTableQualityPolicy( patternName ).then(() => getPatterns()); }; + if (loading) { + return ( +
+ +
+ ); + } + return (
diff --git a/dqops/src/main/frontend/src/pages/DefaultSchedulesDetail/index.tsx b/dqops/src/main/frontend/src/pages/DefaultSchedulesDetail/index.tsx index 03a5fd1a87..378c571769 100644 --- a/dqops/src/main/frontend/src/pages/DefaultSchedulesDetail/index.tsx +++ b/dqops/src/main/frontend/src/pages/DefaultSchedulesDetail/index.tsx @@ -1,15 +1,14 @@ import React, { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; import { useHistory, useLocation } from 'react-router-dom'; import { MonitoringScheduleSpec } from '../../api'; import Button from '../../components/Button'; import ScheduleView from '../../components/ScheduleView'; -import SvgIcon from '../../components/SvgIcon'; import Tabs from '../../components/Tabs'; +import { IRootState } from '../../redux/reducers'; import { SettingsApi } from '../../services/apiClient'; import { CheckRunMonitoringScheduleGroup } from '../../shared/enums/scheduling.enum'; import { useDecodedParams } from '../../utils'; -import { useSelector } from 'react-redux'; -import { IRootState } from '../../redux/reducers'; const tabs = [ { @@ -96,36 +95,34 @@ const DefaultSchedulesDetail = () => { return ( <> -
-
-
- -
- Default schedule editor -
+
+
+
+ Default schedule editor
-
-
-
+
+
+ +
+
(); const [isUpdated, setIsUpdated] = useState(false); + const [loading, setLoading] = useState(false); const dispatch = useActionDispatch(); useEffect(() => { if (!incidentFilters) return; @@ -80,8 +82,9 @@ export default function DefaultWebhooksDetail() { }; const getFilteredNotifications = () => { - FilteredNotificationsConfigurationsClient.getDefaultFilteredNotificationsConfigurations().then( - (response) => { + setLoading(true); + FilteredNotificationsConfigurationsClient.getDefaultFilteredNotificationsConfigurations() + .then((response) => { const patterns: TNotificationPattern[] = response.data.map((x) => { return { ...x, @@ -94,9 +97,11 @@ export default function DefaultWebhooksDetail() { checkType: x.filter?.checkType || '' }; }); - setFilteredNotifications(patterns); - } - ); + const sortedPatterns = sortPatterns(patterns, 'priority', 'asc'); + + setFilteredNotifications(sortedPatterns); + }) + .finally(() => setLoading(false)); }; useEffect(() => { @@ -106,11 +111,15 @@ export default function DefaultWebhooksDetail() { const createNotificationPattern = () => { setAddNotificationPattern(true); - dispatch(updateTabLabel('New default notification', activeTab ?? '')); + dispatch(updateTabLabel('New filtered notification', activeTab ?? '')); }; const onBack = () => { - dispatch(updateTabLabel('Default notifications', activeTab ?? '')); + dispatch( + updateTabLabel('Global incident notifications', activeTab ?? '', { + incidentFilters: undefined + }) + ); setPatternNameEdit(''); setAddNotificationPattern(false); @@ -152,8 +161,14 @@ export default function DefaultWebhooksDetail() {
- Default incident notification configuration + Global incident notification configuration
+
@@ -161,12 +176,7 @@ export default function DefaultWebhooksDetail() { filteredNotificationsConfigurations={filteredNotifications} onChange={setFilteredNotifications} setPatternNameEdit={onCHangePatternNameToEdit} - /> -
diff --git a/dqops/src/main/frontend/src/pages/Home/StaticHomePage.tsx b/dqops/src/main/frontend/src/pages/Home/StaticHomePage.tsx index 63b735be4a..5726004116 100644 --- a/dqops/src/main/frontend/src/pages/Home/StaticHomePage.tsx +++ b/dqops/src/main/frontend/src/pages/Home/StaticHomePage.tsx @@ -1,10 +1,23 @@ -import React from 'react'; +import React, { useEffect } from 'react'; +import { EnviromentApiClient } from '../../services/apiClient'; export default function StaticHomePage() { + const [title, setTitle] = React.useState(''); + const [showLinks, setShowLinks] = React.useState(false); + + useEffect(() => { + EnviromentApiClient.getDqoSettings().then((res) => { + setTitle(String(res.data?.properties?.['dqo.ui.application-name'])); + setShowLinks( + String(res.data?.properties?.['dqo.ui.home-page.show-links']) === 'true' + ); + }); + }, []); + return (
- Welcome to DQOps Data Quality Operations Center + {title ? `Welcome to ${title}` : <> }

@@ -143,40 +156,42 @@ export default function StaticHomePage() {

-
-

Check the docs for more tutorials

- - - -

- Download our best practices for effective
- data quality improvement -

- - - + {showLinks && ( +
+

Check the docs for more tutorials

+ + + +

+ Download our best practices for effective
+ data quality improvement +

+ + + -

Contact us for more information

- - Contact us - +

Contact us for more information

+ + Contact us + -

Check our progress on GitHub

- - - -

Visit DQOps website

- - - -
+

Check our progress on GitHub

+ + + +

Visit DQOps website

+ + + +
+ )}
diff --git a/dqops/src/main/frontend/src/pages/Home/index.tsx b/dqops/src/main/frontend/src/pages/Home/index.tsx index 10a87973fa..cf28247c1b 100644 --- a/dqops/src/main/frontend/src/pages/Home/index.tsx +++ b/dqops/src/main/frontend/src/pages/Home/index.tsx @@ -6,15 +6,16 @@ import Tabs from '../../components/Tabs'; import { useActionDispatch } from '../../hooks/useActionDispatch'; import { setHomeFirstLevelTab } from '../../redux/actions/source.actions'; import { IRootState } from '../../redux/reducers'; -import ColumnListView from '../ColumnListView/ColumnListView'; +import DataQualitySummary from '../DataQualitySummary'; import GlobalIncidents from '../GlobalIncidents'; -import TableListView from '../TableListView/TableListView'; import StaticHomePage from './StaticHomePage'; const tabs = [ { label: 'Home', value: '/home' }, - { label: 'Tables', value: '/tables' }, - { label: 'Columns', value: '/columns' }, + { + label: 'Data quality summary', + value: '/data-quality-summary' + }, { label: 'Incidents summary', value: '/global-incidents' @@ -59,8 +60,7 @@ const HomePage = () => {
{activeTab === '/home' && } - {activeTab === '/tables' && } - {activeTab === '/columns' && } + {activeTab === '/data-quality-summary' && } {activeTab === '/global-incidents' && } {/* {activeTab === '/global-incidents?groupBy=category' && ( diff --git a/dqops/src/main/frontend/src/pages/IncidentConnection/index.tsx b/dqops/src/main/frontend/src/pages/IncidentConnection/index.tsx index 95fe76d24b..fbc4f51d6e 100644 --- a/dqops/src/main/frontend/src/pages/IncidentConnection/index.tsx +++ b/dqops/src/main/frontend/src/pages/IncidentConnection/index.tsx @@ -337,7 +337,7 @@ export const IncidentConnection = () => { return ( openIncidentDetail(row)} > {values.join(', ')} (more) @@ -403,7 +403,7 @@ export const IncidentConnection = () => { href={value} target="_blank" rel="noreferrer" - className="text-blue-600 underline" + className="underline" > { addIssueUrl(row)} @@ -426,6 +427,7 @@ export const IncidentConnection = () => {
) : ( addIssueUrl(row)} @@ -486,7 +488,7 @@ export const IncidentConnection = () => { url: ROUTES.CONNECTION_DETAIL( CheckTypes.SOURCES, connection, - 'incidents' + 'incidents-grouping' ), value: ROUTES.CONNECTION_LEVEL_VALUE(CheckTypes.SOURCES, connection), state: {}, @@ -494,7 +496,11 @@ export const IncidentConnection = () => { }) ); history.push( - ROUTES.CONNECTION_DETAIL(CheckTypes.SOURCES, connection, 'incidents') + ROUTES.CONNECTION_DETAIL( + CheckTypes.SOURCES, + connection, + 'incidents-grouping' + ) ); }; diff --git a/dqops/src/main/frontend/src/pages/IncidentDetail/IncidentIssueList.tsx b/dqops/src/main/frontend/src/pages/IncidentDetail/IncidentIssueList.tsx index 20cbeb4ea1..a0a19056d0 100644 --- a/dqops/src/main/frontend/src/pages/IncidentDetail/IncidentIssueList.tsx +++ b/dqops/src/main/frontend/src/pages/IncidentDetail/IncidentIssueList.tsx @@ -155,10 +155,7 @@ export const IncidentIssueRow = ({ ) : null} {doesColumnHaveValues('checkName') ? ( - + {issue.checkName} diff --git a/dqops/src/main/frontend/src/pages/IncidentDetail/index.tsx b/dqops/src/main/frontend/src/pages/IncidentDetail/index.tsx index 9ca4d976bf..a4b2046462 100644 --- a/dqops/src/main/frontend/src/pages/IncidentDetail/index.tsx +++ b/dqops/src/main/frontend/src/pages/IncidentDetail/index.tsx @@ -280,7 +280,7 @@ export const IncidentDetail = () => { const url = ROUTES.CONNECTION_DETAIL( CheckTypes.SOURCES, connection, - 'incidents' + 'notifications' ); dispatch( addFirstLevelTab(CheckTypes.SOURCES, { @@ -582,7 +582,7 @@ export const IncidentDetail = () => { href={incidentDetail?.issueUrl} target="_blank" rel="noreferrer" - className="text-blue-600 underline" + className="underline" > { setOpen(true)} @@ -608,6 +609,7 @@ export const IncidentDetail = () => {
) : ( setOpen(true)} @@ -720,7 +722,7 @@ export const IncidentDetail = () => { open={createNotificationDialogOpen} onClose={() => setCreateNotificationDialogOpen(false)} onConfirm={createConfirmNotification} - message="No notification filters are defined for this incident, do you want to create a notification configuration for incidents similar to this incident?" + message="No notification filters have been defined for this incident. Would you like to create a notification configuration based on this incident?" yesNo /> diff --git a/dqops/src/main/frontend/src/pages/RuleDetail/index.tsx b/dqops/src/main/frontend/src/pages/RuleDetail/index.tsx index f38caf55f0..a88c0eaea5 100644 --- a/dqops/src/main/frontend/src/pages/RuleDetail/index.tsx +++ b/dqops/src/main/frontend/src/pages/RuleDetail/index.tsx @@ -2,6 +2,7 @@ import React, { ChangeEvent, useEffect, useState } from 'react'; import { useSelector } from 'react-redux'; import ConfirmDialog from '../../components/CustomTree/ConfirmDialog'; import Input from '../../components/Input'; +import Loader from '../../components/Loader'; import { RuleActionGroup } from '../../components/Sensors/RuleActionGroup'; import SvgIcon from '../../components/SvgIcon'; import Tabs from '../../components/Tabs'; @@ -36,8 +37,11 @@ export const RuleDetail = () => { const { full_rule_name, ruleDetail, path, type, copied } = useSelector( getFirstLevelSensorState ); - const { refreshRulesTreeIndicator, activeTab: firstLevelActiveTab } = - useSelector((state: IRootState) => state.definition); + const { + refreshRulesTreeIndicator, + activeTab: firstLevelActiveTab, + loading + } = useSelector((state: IRootState) => state.definition); const dispatch = useActionDispatch(); const [activeTab, setActiveTab] = useState('definition'); const [ruleName, setRuleName] = useState( @@ -197,7 +201,13 @@ export const RuleDetail = () => { closeRuleFirstLevelTab() ); }; - + if (loading) { + return ( +
+ +
+ ); + } return ( <>
@@ -230,6 +240,8 @@ export const RuleDetail = () => { value={ruleName} onChange={onChangeRuleName} error={!ruleName} + className="!min-w-100" + autoFocus />
diff --git a/dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx b/dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx index 3f04eb0d1c..65e3a83280 100644 --- a/dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx +++ b/dqops/src/main/frontend/src/pages/Schema/SchemaTableItem.tsx @@ -86,7 +86,7 @@ export default function SchemaTableItem({ checkTypes ?? CheckTypes.MONITORING, item.connection_name ?? '', item.target?.schema_name ?? '', - 'tables' + 'data-quality-summary' ); dispatch( addFirstLevelTab(checkTypes ?? CheckTypes.MONITORING, { diff --git a/dqops/src/main/frontend/src/pages/Schema/SchemaTableItemDimensions.tsx b/dqops/src/main/frontend/src/pages/Schema/SchemaTableItemDimensions.tsx index 948da3fb1f..19c5fd35cd 100644 --- a/dqops/src/main/frontend/src/pages/Schema/SchemaTableItemDimensions.tsx +++ b/dqops/src/main/frontend/src/pages/Schema/SchemaTableItemDimensions.tsx @@ -208,9 +208,10 @@ const renderSecondLevelTooltip = (
Data quality KPI:
- {data?.data_quality_kpi - ? Number(data?.data_quality_kpi) === 100 - ? '100%' + {data?.data_quality_kpi || data?.data_quality_kpi === 0 + ? Number(data?.data_quality_kpi) === 100 || + data?.data_quality_kpi === 0 + ? Number(data?.data_quality_kpi).toFixed(0) + '%' : Number(data?.data_quality_kpi).toFixed(2) + '%' : '-'}
diff --git a/dqops/src/main/frontend/src/pages/Schema/index.tsx b/dqops/src/main/frontend/src/pages/Schema/index.tsx index 1a3877c303..c0b641b72e 100644 --- a/dqops/src/main/frontend/src/pages/Schema/index.tsx +++ b/dqops/src/main/frontend/src/pages/Schema/index.tsx @@ -12,8 +12,7 @@ import { IRootState } from '../../redux/reducers'; import { getFirstLevelActiveTab } from '../../redux/selectors'; import { CheckTypes, ROUTES } from '../../shared/routes'; import { useDecodedParams } from '../../utils'; -import ColumnListView from '../ColumnListView/ColumnListView'; -import TableListView from '../TableListView/TableListView'; +import DataQualitySummary from '../DataQualitySummary'; import { MultiChecks } from './MultiCheck/MultiChecks'; const SchemaPage = () => { @@ -38,12 +37,8 @@ const SchemaPage = () => { const tabs = useMemo( () => [ { - label: 'Tables', - value: 'tables' - }, - { - label: 'Columns', - value: 'columns' + label: 'Data quality summary', + value: 'data-quality-summary' }, ...(checkTypes !== CheckTypes.SOURCES ? [ @@ -116,16 +111,7 @@ const SchemaPage = () => {
- {activeTab === 'tables' && ( -
- -
- )} - {activeTab === 'columns' && ( -
- -
- )} + {activeTab === 'data-quality-summary' && } {activeTab === 'import-tables' && } {checkTypes !== CheckTypes.SOURCES && activeTab === 'multiple_checks' && ( diff --git a/dqops/src/main/frontend/src/pages/SensorDetail/index.tsx b/dqops/src/main/frontend/src/pages/SensorDetail/index.tsx index ce0d3d00fa..5c15f7498a 100644 --- a/dqops/src/main/frontend/src/pages/SensorDetail/index.tsx +++ b/dqops/src/main/frontend/src/pages/SensorDetail/index.tsx @@ -6,6 +6,7 @@ import { } from '../../api'; import ConfirmDialog from '../../components/CustomTree/ConfirmDialog'; import Input from '../../components/Input'; +import Loader from '../../components/Loader'; import { SensorActionGroup } from '../../components/Sensors/SensorActionGroup'; import SvgIcon from '../../components/SvgIcon'; import Tabs from '../../components/Tabs'; @@ -84,8 +85,11 @@ export const SensorDetail = () => { const { full_sensor_name, sensorDetail, path, type, copied } = useSelector( getFirstLevelSensorState ); - const { refreshSensorsTreeIndicator, activeTab: firstLevelActiveTab } = - useSelector((state: IRootState) => state.definition); + const { + refreshSensorsTreeIndicator, + activeTab: firstLevelActiveTab, + loading + } = useSelector((state: IRootState) => state.definition); const dispatch = useActionDispatch(); const [activeTab, setActiveTab] = useState('definition'); const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); @@ -277,6 +281,14 @@ export const SensorDetail = () => { ); }; + if (loading) { + return ( +
+ +
+ ); + } + return ( <>
@@ -308,6 +320,8 @@ export const SensorDetail = () => { value={sensorName} onChange={onChangeSensorName} error={!sensorName} + className="!min-w-100" + autoFocus />
diff --git a/dqops/src/main/frontend/src/pages/SharedCredentialsDetail/SharedCredentailTable.tsx b/dqops/src/main/frontend/src/pages/SharedCredentialsDetail/SharedCredentailTable.tsx index 4662a80786..a97ae1fea0 100644 --- a/dqops/src/main/frontend/src/pages/SharedCredentialsDetail/SharedCredentailTable.tsx +++ b/dqops/src/main/frontend/src/pages/SharedCredentialsDetail/SharedCredentailTable.tsx @@ -1,88 +1,111 @@ +import { IconButton } from '@material-tailwind/react'; import React, { useState } from 'react'; +import { useSelector } from 'react-redux'; import { SharedCredentialListModel } from '../../api'; -import Button from '../../components/Button'; import ConfirmDialog from '../../components/CustomTree/ConfirmDialog'; -import { useActionDispatch } from '../../hooks/useActionDispatch'; -import { addFirstLevelTab } from '../../redux/actions/definition.actions'; -import { ROUTES } from '../../shared/routes'; +import ClientSidePagination from '../../components/Pagination/ClientSidePagination'; // Import the pagination component +import SvgIcon from '../../components/SvgIcon'; +import { IRootState } from '../../redux/reducers'; type TSharedCredentialTableProps = { sharedCredentialList: SharedCredentialListModel[]; deleteSharedCredential: (credential: string) => Promise; + setCredentialToEdit: (credential: string) => void; filter?: boolean; }; -export default function SharedCredentailTable({ +export default function SharedCredentialTable({ sharedCredentialList, - deleteSharedCredential + deleteSharedCredential, + setCredentialToEdit }: TSharedCredentialTableProps) { + const { userProfile } = useSelector((state: IRootState) => state.job || {}); + const [ selectedSharedCredentialToDelete, setSelectedSharedCredentialToDelete ] = useState(''); - const dispatch = useActionDispatch(); + + const [displayedCredentials, setDisplayedCredentials] = useState< + SharedCredentialListModel[] + >([]); + const updateSharedCredential = async (credential_name: string) => { - dispatch( - addFirstLevelTab({ - url: ROUTES.SHARED_CREDENTIALS_DETAIL(credential_name), - value: ROUTES.SHARED_CREDENTIALS_DETAIL_VALUE(credential_name), - label: credential_name, - state: { - credential_name - } - }) - ); + setCredentialToEdit(credential_name); }; return ( - - {sharedCredentialList?.map((credential, index) => ( - - - updateSharedCredential(credential.credential_name ?? '') - } - > - {credential.credential_name} - - {credential.type} - -
+
-
Type of credential:
+
Type of credential:
state.job || {}); - const dispatch = useActionDispatch() - - const [sharedCredentialList, setSharedCredentialList] = useState(); - const [loading, setLoading] = useState(false) + const { activeTab } = useSelector((state: IRootState) => state.definition); - const getSharedCredentialList = async () => { + const dispatch = useActionDispatch(); + + const [sharedCredentialList, setSharedCredentialList] = + useState(); + const [loading, setLoading] = useState(false); + const [credentialToEdit, setCredentialToEdit] = React.useState< + string | undefined + >(undefined); + const [createCredential, setCreateCredential] = + React.useState(false); + const onBack = () => { + setCredentialToEdit(undefined); + setCreateCredential(false); + dispatch(updateTabLabel('Shared credentials', activeTab ?? '')); + getSharedCredentialList(); + }; + const getSharedCredentialList = async () => { await SharedCredentialsApi.getAllSharedCredentials() .then((res) => setSharedCredentialList(res.data)) .catch((err) => console.error(err)) - .finally(() => setLoading(false)) - } + .finally(() => setLoading(false)); + }; const addSharedCredential = async () => { - dispatch( - addFirstLevelTab({ - url: ROUTES.SHARED_CREDENTIALS_DETAIL("new"), - value: ROUTES.SHARED_CREDENTIALS_DETAIL_VALUE("new"), - label: "Add credential", - }) - ); - } + setCreateCredential(true); + }; const deleteSharedCredential = async (credential: string) => { await SharedCredentialsApi.deleteSharedCredential(credential) - .then(() => { - setLoading(true) - getSharedCredentialList() - }) - .catch((err) => console.error(err)) - - } + .then(() => { + setLoading(true); + getSharedCredentialList(); + }) + .catch((err) => console.error(err)); + }; useEffect(() => { - setLoading(true) - getSharedCredentialList() - }, []) - + setLoading(true); + getSharedCredentialList(); + }, []); if (loading) { - return( - <> -
- + return ( + <> +
+
- -)} - + + ); + } return ( <> - {userProfile.can_manage_and_view_shared_credentials === true ? - - - - - - + {credentialToEdit || createCredential ? ( + + ) : userProfile.can_manage_and_view_shared_credentials === true ? ( +
Credential nameCredential type
+ + + + + +
Credential nameCredential typeAction
- : -
- Access denied -
- } + ) : ( +
+ Access denied +
+ )} - ) + ); } diff --git a/dqops/src/main/frontend/src/pages/Table/index.tsx b/dqops/src/main/frontend/src/pages/Table/index.tsx index 0991519854..9cf8e62593 100644 --- a/dqops/src/main/frontend/src/pages/Table/index.tsx +++ b/dqops/src/main/frontend/src/pages/Table/index.tsx @@ -5,6 +5,7 @@ import MonitoringView from '../../components/Connection/TableView/MonitoringView import PartitionedChecks from '../../components/Connection/TableView/PartitionedChecks'; import ProfilingView from '../../components/Connection/TableView/ProfilingView'; import ScheduleDetail from '../../components/Connection/TableView/ScheduleDetail'; +import SourceTables from '../../components/Connection/TableView/SourceTables/SourceTables'; import TableCommentView from '../../components/Connection/TableView/TableCommentView'; import TableDataGroupingConfiguration from '../../components/Connection/TableView/TableDataGroupingConfigurations'; import TableDetails from '../../components/Connection/TableView/TableDetails'; @@ -51,6 +52,10 @@ const initTabs = [ { label: 'Incident configuration', value: 'incident_configuration' + }, + { + label: 'Data lineage', + value: 'source_tables' } ]; @@ -266,6 +271,7 @@ const TablePage = () => { )}
+
{activeTab === 'source_tables' && }
)}
diff --git a/dqops/src/main/frontend/src/pages/TableColumnsView/TableColumnsBody.tsx b/dqops/src/main/frontend/src/pages/TableColumnsView/TableColumnsBody.tsx index 8db2d44cfd..ddb338355b 100644 --- a/dqops/src/main/frontend/src/pages/TableColumnsView/TableColumnsBody.tsx +++ b/dqops/src/main/frontend/src/pages/TableColumnsView/TableColumnsBody.tsx @@ -1,17 +1,13 @@ -import { IconButton, Tooltip } from '@material-tailwind/react'; -import clsx from 'clsx'; -import moment from 'moment'; +import { IconButton } from '@material-tailwind/react'; import React, { useEffect, useMemo, useState } from 'react'; import { useDispatch, useSelector } from 'react-redux'; import { useHistory } from 'react-router-dom'; import { - DimensionCurrentDataQualityStatusModel, - DimensionCurrentDataQualityStatusModelCurrentSeverityEnum, DqoJobHistoryEntryModelStatusEnum, TableColumnsStatisticsModel } from '../../api'; import Checkbox from '../../components/Checkbox'; -import { getColor } from '../../components/Connection/TableView/TableQualityStatus/TableQualityStatusUtils'; +import QualityDimensionStatuses from '../../components/DataQualityChecks/QualityDimension/QualityDimensionStatuses'; import SvgIcon from '../../components/SvgIcon'; import { addFirstLevelTab } from '../../redux/actions/source.actions'; import { IRootState } from '../../redux/reducers'; @@ -88,6 +84,8 @@ export default function TableColumnsBody({ statistics?.column_statistics.map(async (x, index) => x.column_hash === hashValue ? await JobApiClient.collectStatisticsOnTable( + undefined, + false, undefined, false, undefined, @@ -135,22 +133,6 @@ export default function TableColumnsBody({ : ([] as string[]); }, [job]); - const getBasicDimmensionsKeys = (column: MyData, type: string) => { - const basicDimensions = Object.keys(column.dimentions ?? {})?.find( - (x) => x === type - ); - return basicDimensions; - }; - const basicDimensionTypes = ['Completeness', 'Validity', 'Consistency']; - - const getAdditionalDimentionsKeys = (column: MyData) => { - return ( - Object.keys(column.dimentions ?? {})?.filter( - (x) => !basicDimensionTypes.includes(x) - ) ?? [] - ); - }; - const getMidSectionItemsBasedOnWidth = () => { const width = window.innerWidth; const excludedItems: string[] = []; @@ -206,73 +188,7 @@ export default function TableColumnsBody({
-
- {basicDimensionTypes.map((dimType) => { - const dimensionKey = getBasicDimmensionsKeys(column, dimType); - const currentSeverity = - column.dimentions?.[dimensionKey as any]?.current_severity; - const lastCheckExecutedAt = - column.dimentions?.[dimensionKey as any] - ?.last_check_executed_at; - const severityColor = getColor(currentSeverity as any); - const hasNoSeverity = severityColor.length === 0; - - const dimensionsClassNames = clsx( - 'w-3 h-3 border border-gray-150', - { - 'bg-gray-150': hasNoSeverity && lastCheckExecutedAt, - [severityColor]: !hasNoSeverity - } - ); - return ( - -
- - ); - })} - {getAdditionalDimentionsKeys(column).map( - (dimensionKey: string, dimIndex) => { - return ( - -
- - ); - } - )} -
+
-
+
{isNaN(Number(column.null_percent)) ? '' @@ -384,6 +300,7 @@ export default function TableColumnsBody({
); } - -const renderSecondLevelTooltip = ( - data: DimensionCurrentDataQualityStatusModel | undefined -) => { - if (data && data.last_check_executed_at) { - return ( -
-
-
Last executed at:
-
- {moment(data?.last_check_executed_at).format('YYYY-MM-DD HH:mm:ss')} -
-
-
-
Current severity level:
-
{data?.current_severity}
-
-
-
Highest historical severity level:
-
{data?.highest_historical_severity}
-
-
-
Quality Dimension:
-
{data?.dimension}
-
-
-
Executed checks:
-
{data?.executed_checks}
-
-
-
Valid results:
-
{data?.valid_results}
-
-
-
Warnings:
-
{data?.warnings}
-
-
-
Errors:
-
{data?.errors}
-
-
-
Fatals:
-
{data?.fatals}
-
-
-
Data quality KPI:
-
- {data.data_quality_kpi !== undefined - ? Number(data.data_quality_kpi).toFixed(2) + ' %' - : '-'} -
-
-
- ); - } - return ( -
-
-
Quality Dimension:
-
{data?.dimension}
-
-
No data quality checks configured
-
- ); -}; diff --git a/dqops/src/main/frontend/src/pages/TableColumnsView/index.tsx b/dqops/src/main/frontend/src/pages/TableColumnsView/index.tsx index 169de591a2..682d5aa7b9 100644 --- a/dqops/src/main/frontend/src/pages/TableColumnsView/index.tsx +++ b/dqops/src/main/frontend/src/pages/TableColumnsView/index.tsx @@ -43,6 +43,7 @@ const TableColumnsView = () => { const dispatch = useActionDispatch(); const history = useHistory(); const [statistics, setStatistics] = useState(); + const [isCollected, setIsCollected] = useState(false); const [checkedColumns, setCheckedColumns] = useState([]); const [jobId, setJobId] = useState(); const firstLevelActiveTab = useSelector(getFirstLevelActiveTab(checkTypes)); @@ -60,7 +61,8 @@ const TableColumnsView = () => { if ( !res.data.column_statistics?.find( (column) => column.statistics && column.statistics.length > 0 - ) + ) && + !isCollected ) { dispatch( setJobAllert({ @@ -80,25 +82,34 @@ const TableColumnsView = () => { const collectStatistics = async () => { dispatch(setJobAllert({})); try { - await JobApiClient.collectStatisticsOnTable(undefined, false, undefined, { + await JobApiClient.collectStatisticsOnTable(undefined, false, undefined, false, undefined, { connection: connectionName, fullTableName: schemaName + '.' + tableName, enabled: true, columnNames: checkedColumns - }).then((res) => setJobId(res.data.jobId?.jobId)); + }).then((res) => { + setIsCollected(true); + dispatch(setJobAllert({})); + setJobId(res.data.jobId?.jobId); + }); } catch (err) { console.error(err); } }; - const filteredJobs = useMemo(() => { + const isCollectingStatistics = useMemo(() => { return ( job && + job.parameters?.collectStatisticsParameters + ?.statistics_collector_search_filters?.connection === connectionName && + job.parameters?.collectStatisticsParameters + ?.statistics_collector_search_filters?.fullTableName === + schemaName + '.' + tableName && (job.status === DqoJobHistoryEntryModelStatusEnum.running || job.status === DqoJobHistoryEntryModelStatusEnum.queued || job.status === DqoJobHistoryEntryModelStatusEnum.waiting) ); - }, [job]); + }, [job, checkTypes, connectionName, schemaName, tableName]); const postDataStream = async () => { const url = ROUTES.TABLE_LEVEL_PAGE( @@ -196,18 +207,18 @@ const TableColumnsView = () => { >